-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
powerpc/mm: Lockless get_user_pages_fast() for 64-bit (v3)
Implement lockless get_user_pages_fast for 64-bit powerpc. Page table existence is guaranteed with RCU, and speculative page references are used to take a reference to the pages without having a prior existence guarantee on them. Signed-off-by: Nick Piggin <npiggin@suse.de> Signed-off-by: Dave Kleikamp <shaggy@linux.vnet.ibm.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
- Loading branch information
Nick Piggin
authored and
Benjamin Herrenschmidt
committed
Jul 30, 2008
1 parent
7d2a175
commit ce0ad7f
Showing
5 changed files
with
310 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,280 @@ | ||
/* | ||
* Lockless get_user_pages_fast for powerpc | ||
* | ||
* Copyright (C) 2008 Nick Piggin | ||
* Copyright (C) 2008 Novell Inc. | ||
*/ | ||
#undef DEBUG | ||
|
||
#include <linux/sched.h> | ||
#include <linux/mm.h> | ||
#include <linux/hugetlb.h> | ||
#include <linux/vmstat.h> | ||
#include <linux/pagemap.h> | ||
#include <linux/rwsem.h> | ||
#include <asm/pgtable.h> | ||
|
||
/* | ||
* The performance critical leaf functions are made noinline otherwise gcc | ||
* inlines everything into a single function which results in too much | ||
* register pressure. | ||
*/ | ||
static noinline int gup_pte_range(pmd_t pmd, unsigned long addr, | ||
unsigned long end, int write, struct page **pages, int *nr) | ||
{ | ||
unsigned long mask, result; | ||
pte_t *ptep; | ||
|
||
result = _PAGE_PRESENT|_PAGE_USER; | ||
if (write) | ||
result |= _PAGE_RW; | ||
mask = result | _PAGE_SPECIAL; | ||
|
||
ptep = pte_offset_kernel(&pmd, addr); | ||
do { | ||
pte_t pte = *ptep; | ||
struct page *page; | ||
|
||
if ((pte_val(pte) & mask) != result) | ||
return 0; | ||
VM_BUG_ON(!pfn_valid(pte_pfn(pte))); | ||
page = pte_page(pte); | ||
if (!page_cache_get_speculative(page)) | ||
return 0; | ||
if (unlikely(pte != *ptep)) { | ||
put_page(page); | ||
return 0; | ||
} | ||
pages[*nr] = page; | ||
(*nr)++; | ||
|
||
} while (ptep++, addr += PAGE_SIZE, addr != end); | ||
|
||
return 1; | ||
} | ||
|
||
#ifdef CONFIG_HUGETLB_PAGE | ||
static noinline int gup_huge_pte(pte_t *ptep, struct hstate *hstate, | ||
unsigned long *addr, unsigned long end, | ||
int write, struct page **pages, int *nr) | ||
{ | ||
unsigned long mask; | ||
unsigned long pte_end; | ||
struct page *head, *page; | ||
pte_t pte; | ||
int refs; | ||
|
||
pte_end = (*addr + huge_page_size(hstate)) & huge_page_mask(hstate); | ||
if (pte_end < end) | ||
end = pte_end; | ||
|
||
pte = *ptep; | ||
mask = _PAGE_PRESENT|_PAGE_USER; | ||
if (write) | ||
mask |= _PAGE_RW; | ||
if ((pte_val(pte) & mask) != mask) | ||
return 0; | ||
/* hugepages are never "special" */ | ||
VM_BUG_ON(!pfn_valid(pte_pfn(pte))); | ||
|
||
refs = 0; | ||
head = pte_page(pte); | ||
page = head + ((*addr & ~huge_page_mask(hstate)) >> PAGE_SHIFT); | ||
do { | ||
VM_BUG_ON(compound_head(page) != head); | ||
pages[*nr] = page; | ||
(*nr)++; | ||
page++; | ||
refs++; | ||
} while (*addr += PAGE_SIZE, *addr != end); | ||
|
||
if (!page_cache_add_speculative(head, refs)) { | ||
*nr -= refs; | ||
return 0; | ||
} | ||
if (unlikely(pte != *ptep)) { | ||
/* Could be optimized better */ | ||
while (*nr) { | ||
put_page(page); | ||
(*nr)--; | ||
} | ||
} | ||
|
||
return 1; | ||
} | ||
#endif /* CONFIG_HUGETLB_PAGE */ | ||
|
||
static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, | ||
int write, struct page **pages, int *nr) | ||
{ | ||
unsigned long next; | ||
pmd_t *pmdp; | ||
|
||
pmdp = pmd_offset(&pud, addr); | ||
do { | ||
pmd_t pmd = *pmdp; | ||
|
||
next = pmd_addr_end(addr, end); | ||
if (pmd_none(pmd)) | ||
return 0; | ||
if (!gup_pte_range(pmd, addr, next, write, pages, nr)) | ||
return 0; | ||
} while (pmdp++, addr = next, addr != end); | ||
|
||
return 1; | ||
} | ||
|
||
static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end, | ||
int write, struct page **pages, int *nr) | ||
{ | ||
unsigned long next; | ||
pud_t *pudp; | ||
|
||
pudp = pud_offset(&pgd, addr); | ||
do { | ||
pud_t pud = *pudp; | ||
|
||
next = pud_addr_end(addr, end); | ||
if (pud_none(pud)) | ||
return 0; | ||
if (!gup_pmd_range(pud, addr, next, write, pages, nr)) | ||
return 0; | ||
} while (pudp++, addr = next, addr != end); | ||
|
||
return 1; | ||
} | ||
|
||
int get_user_pages_fast(unsigned long start, int nr_pages, int write, | ||
struct page **pages) | ||
{ | ||
struct mm_struct *mm = current->mm; | ||
unsigned long addr, len, end; | ||
unsigned long next; | ||
pgd_t *pgdp; | ||
int psize, nr = 0; | ||
unsigned int shift; | ||
|
||
pr_debug("%s(%lx,%x,%s)\n", __func__, start, nr_pages, write ? "write" : "read"); | ||
|
||
start &= PAGE_MASK; | ||
addr = start; | ||
len = (unsigned long) nr_pages << PAGE_SHIFT; | ||
end = start + len; | ||
|
||
if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ, | ||
start, len))) | ||
goto slow_irqon; | ||
|
||
pr_debug(" aligned: %lx .. %lx\n", start, end); | ||
|
||
#ifdef CONFIG_HUGETLB_PAGE | ||
/* We bail out on slice boundary crossing when hugetlb is | ||
* enabled in order to not have to deal with two different | ||
* page table formats | ||
*/ | ||
if (addr < SLICE_LOW_TOP) { | ||
if (end > SLICE_LOW_TOP) | ||
goto slow_irqon; | ||
|
||
if (unlikely(GET_LOW_SLICE_INDEX(addr) != | ||
GET_LOW_SLICE_INDEX(end - 1))) | ||
goto slow_irqon; | ||
} else { | ||
if (unlikely(GET_HIGH_SLICE_INDEX(addr) != | ||
GET_HIGH_SLICE_INDEX(end - 1))) | ||
goto slow_irqon; | ||
} | ||
#endif /* CONFIG_HUGETLB_PAGE */ | ||
|
||
/* | ||
* XXX: batch / limit 'nr', to avoid large irq off latency | ||
* needs some instrumenting to determine the common sizes used by | ||
* important workloads (eg. DB2), and whether limiting the batch size | ||
* will decrease performance. | ||
* | ||
* It seems like we're in the clear for the moment. Direct-IO is | ||
* the main guy that batches up lots of get_user_pages, and even | ||
* they are limited to 64-at-a-time which is not so many. | ||
*/ | ||
/* | ||
* This doesn't prevent pagetable teardown, but does prevent | ||
* the pagetables from being freed on powerpc. | ||
* | ||
* So long as we atomically load page table pointers versus teardown, | ||
* we can follow the address down to the the page and take a ref on it. | ||
*/ | ||
local_irq_disable(); | ||
|
||
psize = get_slice_psize(mm, addr); | ||
shift = mmu_psize_defs[psize].shift; | ||
|
||
#ifdef CONFIG_HUGETLB_PAGE | ||
if (unlikely(mmu_huge_psizes[psize])) { | ||
pte_t *ptep; | ||
unsigned long a = addr; | ||
unsigned long sz = ((1UL) << shift); | ||
struct hstate *hstate = size_to_hstate(sz); | ||
|
||
BUG_ON(!hstate); | ||
/* | ||
* XXX: could be optimized to avoid hstate | ||
* lookup entirely (just use shift) | ||
*/ | ||
|
||
do { | ||
VM_BUG_ON(shift != mmu_psize_defs[get_slice_psize(mm, a)].shift); | ||
ptep = huge_pte_offset(mm, a); | ||
pr_debug(" %016lx: huge ptep %p\n", a, ptep); | ||
if (!ptep || !gup_huge_pte(ptep, hstate, &a, end, write, pages, | ||
&nr)) | ||
goto slow; | ||
} while (a != end); | ||
} else | ||
#endif /* CONFIG_HUGETLB_PAGE */ | ||
{ | ||
pgdp = pgd_offset(mm, addr); | ||
do { | ||
pgd_t pgd = *pgdp; | ||
|
||
VM_BUG_ON(shift != mmu_psize_defs[get_slice_psize(mm, addr)].shift); | ||
pr_debug(" %016lx: normal pgd %p\n", addr, (void *)pgd); | ||
next = pgd_addr_end(addr, end); | ||
if (pgd_none(pgd)) | ||
goto slow; | ||
if (!gup_pud_range(pgd, addr, next, write, pages, &nr)) | ||
goto slow; | ||
} while (pgdp++, addr = next, addr != end); | ||
} | ||
local_irq_enable(); | ||
|
||
VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT); | ||
return nr; | ||
|
||
{ | ||
int ret; | ||
|
||
slow: | ||
local_irq_enable(); | ||
slow_irqon: | ||
pr_debug(" slow path ! nr = %d\n", nr); | ||
|
||
/* Try to get the remaining pages with get_user_pages */ | ||
start += nr << PAGE_SHIFT; | ||
pages += nr; | ||
|
||
down_read(&mm->mmap_sem); | ||
ret = get_user_pages(current, mm, start, | ||
(end - start) >> PAGE_SHIFT, write, 0, pages, NULL); | ||
up_read(&mm->mmap_sem); | ||
|
||
/* Have to be a bit careful with return values */ | ||
if (nr > 0) { | ||
if (ret < 0) | ||
ret = nr; | ||
else | ||
ret += nr; | ||
} | ||
|
||
return ret; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters