Skip to content

Commit

Permalink
mm/gup: add FOLL_UNLOCKABLE
Browse files Browse the repository at this point in the history 
Setting FOLL_UNLOCKABLE allows GUP to lock/unlock the mmap lock on its
own.  It is a more explicit replacement for locked != NULL.  This clears
the way for passing in locked = 1, without intending that the lock can be
unlocked.

Set the flag in all cases where it is used, eg locked is present in the
external interface or locked is used internally with locked = 0.

Link: https://lkml.kernel.org/r/8-v2-987e91b59705+36b-gup_tidy_jgg@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Acked-by: Mike Rapoport (IBM) <rppt@kernel.org>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Claudio Imbrenda <imbrenda@linux.ibm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  • Loading branch information
Jason Gunthorpe authored and Andrew Morton committed Feb 10, 2023
1 parent 6e4382c commit f04740f
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 13 deletions.
1 change: 1 addition & 0 deletions include/linux/mm_types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1104,5 +1104,6 @@ typedef unsigned int __bitwise zap_flags_t;
#define FOLL_FAST_ONLY 0x80000 /* gup_fast: prevent fall-back to slow gup */
#define FOLL_PCI_P2PDMA 0x100000 /* allow returning PCI P2PDMA pages */
#define FOLL_INTERRUPTIBLE 0x200000 /* allow interrupts from generic signals */
#define FOLL_UNLOCKABLE 0x400000 /* allow unlocking the mmap lock (internal only) */

#endif /* _LINUX_MM_TYPES_H */
36 changes: 23 additions & 13 deletions mm/gup.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -896,7 +896,7 @@ static int faultin_page(struct vm_area_struct *vma,
fault_flags |= FAULT_FLAG_WRITE;
if (*flags & FOLL_REMOTE)
fault_flags |= FAULT_FLAG_REMOTE;
if (locked) {
if (*flags & FOLL_UNLOCKABLE) {
fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
/*
* FAULT_FLAG_INTERRUPTIBLE is opt-in. GUP callers must set
Expand Down Expand Up @@ -1382,9 +1382,11 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm,
for (;;) {
ret = __get_user_pages(mm, start, nr_pages, flags, pages,
vmas, locked);
if (!locked)
if (!(flags & FOLL_UNLOCKABLE)) {
/* VM_FAULT_RETRY couldn't trigger, bypass */
return ret;
pages_done = ret;
break;
}

/* VM_FAULT_RETRY or VM_FAULT_COMPLETED cannot return errors */
if (!*locked) {
Expand Down Expand Up @@ -1532,6 +1534,9 @@ long populate_vma_page_range(struct vm_area_struct *vma,
if (vma_is_accessible(vma))
gup_flags |= FOLL_FORCE;

if (locked)
gup_flags |= FOLL_UNLOCKABLE;

/*
* We made sure addr is within a VMA, so the following will
* not result in a stack expansion that recurses back here.
Expand Down Expand Up @@ -1583,7 +1588,7 @@ long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start,
* a poisoned page.
* !FOLL_FORCE: Require proper access permissions.
*/
gup_flags = FOLL_TOUCH | FOLL_HWPOISON;
gup_flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_UNLOCKABLE;
if (write)
gup_flags |= FOLL_WRITE;

Expand Down Expand Up @@ -2107,12 +2112,20 @@ static bool is_valid_gup_args(struct page **pages, struct vm_area_struct **vmas,
* interfaces:
* - FOLL_PIN/FOLL_TRIED/FOLL_FAST_ONLY are internal only
* - FOLL_REMOTE is internal only and used on follow_page()
* - FOLL_UNLOCKABLE is internal only and used if locked is !NULL
*/
if (WARN_ON_ONCE(gup_flags & (FOLL_PIN | FOLL_TRIED |
if (WARN_ON_ONCE(gup_flags & (FOLL_PIN | FOLL_TRIED | FOLL_UNLOCKABLE |
FOLL_REMOTE | FOLL_FAST_ONLY)))
return false;

gup_flags |= to_set;
if (locked) {
/* At the external interface locked must be set */
if (WARN_ON_ONCE(*locked != 1))
return false;

gup_flags |= FOLL_UNLOCKABLE;
}

/* FOLL_GET and FOLL_PIN are mutually exclusive. */
if (WARN_ON_ONCE((gup_flags & (FOLL_PIN | FOLL_GET)) ==
Expand All @@ -2127,10 +2140,6 @@ static bool is_valid_gup_args(struct page **pages, struct vm_area_struct **vmas,
if (WARN_ON_ONCE((gup_flags & (FOLL_GET | FOLL_PIN)) && !pages))
return false;

/* At the external interface locked must be set */
if (WARN_ON_ONCE(locked && *locked != 1))
return false;

/* We want to allow the pgmap to be hot-unplugged at all times */
if (WARN_ON_ONCE((gup_flags & FOLL_LONGTERM) &&
(gup_flags & FOLL_PCI_P2PDMA)))
Expand All @@ -2140,7 +2149,7 @@ static bool is_valid_gup_args(struct page **pages, struct vm_area_struct **vmas,
* Can't use VMAs with locked, as locked allows GUP to unlock
* which invalidates the vmas array
*/
if (WARN_ON_ONCE(vmas && locked))
if (WARN_ON_ONCE(vmas && (gup_flags & FOLL_UNLOCKABLE)))
return false;

*gup_flags_p = gup_flags;
Expand Down Expand Up @@ -2280,7 +2289,8 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
{
int locked = 0;

if (!is_valid_gup_args(pages, NULL, NULL, &gup_flags, FOLL_TOUCH))
if (!is_valid_gup_args(pages, NULL, NULL, &gup_flags,
FOLL_TOUCH | FOLL_UNLOCKABLE))
return -EINVAL;

return __get_user_pages_locked(current->mm, start, nr_pages, pages,
Expand Down Expand Up @@ -2968,7 +2978,7 @@ static int internal_get_user_pages_fast(unsigned long start,
pages += nr_pinned;
ret = __gup_longterm_locked(current->mm, start, nr_pages - nr_pinned,
pages, NULL, &locked,
gup_flags | FOLL_TOUCH);
gup_flags | FOLL_TOUCH | FOLL_UNLOCKABLE);
if (ret < 0) {
/*
* The caller has to unpin the pages we already pinned so
Expand Down Expand Up @@ -3195,7 +3205,7 @@ long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
int locked = 0;

if (!is_valid_gup_args(pages, NULL, NULL, &gup_flags,
FOLL_PIN | FOLL_TOUCH))
FOLL_PIN | FOLL_TOUCH | FOLL_UNLOCKABLE))
return 0;

return __gup_longterm_locked(current->mm, start, nr_pages, pages, NULL,
Expand Down

0 comments on commit f04740f

Please sign in to comment.