Skip to content

Commit

Permalink
mm/mmap: use maple tree for unmapped_area{_topdown}
Browse files Browse the repository at this point in the history 
The maple tree code was added to find the unmapped area in a previous
commit and was checked against what the rbtree returned, but the actual
result was never used.  Start using the maple tree implementation and
remove the rbtree code.

Add kernel documentation comment for these functions.

Link: https://lkml.kernel.org/r/20220906194824.2110408-14-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Tested-by: Yu Zhao <yuzhao@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: "Matthew Wilcox (Oracle)" <willy@infradead.org>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  • Loading branch information
Liam R. Howlett authored and Andrew Morton committed Sep 27, 2022
1 parent 7fdbd37 commit 3499a13
Showing 1 changed file with 34 additions and 221 deletions.
255 changes: 34 additions & 221 deletions mm/mmap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2013,250 +2013,63 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
return error;
}

/**
* unmapped_area() - Find an area between the low_limit and the high_limit with
* the correct alignment and offset, all from @info. Note: current->mm is used
* for the search.
*
* @info: The unmapped area information including the range (low_limit -
* hight_limit), the alignment offset and mask.
*
* Return: A memory address or -ENOMEM.
*/
static unsigned long unmapped_area(struct vm_unmapped_area_info *info)
{
/*
* We implement the search by looking for an rbtree node that
* immediately follows a suitable gap. That is,
* - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
* - gap_end = vma->vm_start >= info->low_limit + length;
* - gap_end - gap_start >= length
*/
unsigned long length, gap;

struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long length, low_limit, high_limit, gap_start, gap_end;
unsigned long gap;
MA_STATE(mas, &mm->mm_mt, 0, 0);
MA_STATE(mas, &current->mm->mm_mt, 0, 0);

/* Adjust search length to account for worst case alignment overhead */
length = info->length + info->align_mask;
if (length < info->length)
return -ENOMEM;

mas_empty_area(&mas, info->low_limit, info->high_limit - 1,
length);
gap = mas.index;
gap += (info->align_offset - gap) & info->align_mask;

/* Adjust search limits by the desired length */
if (info->high_limit < length)
if (mas_empty_area(&mas, info->low_limit, info->high_limit - 1,
length))
return -ENOMEM;
high_limit = info->high_limit - length;

if (info->low_limit > high_limit)
return -ENOMEM;
low_limit = info->low_limit + length;

/* Check if rbtree root looks promising */
if (RB_EMPTY_ROOT(&mm->mm_rb))
goto check_highest;
vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
if (vma->rb_subtree_gap < length)
goto check_highest;

while (true) {
/* Visit left subtree if it looks promising */
gap_end = vm_start_gap(vma);
if (gap_end >= low_limit && vma->vm_rb.rb_left) {
struct vm_area_struct *left =
rb_entry(vma->vm_rb.rb_left,
struct vm_area_struct, vm_rb);
if (left->rb_subtree_gap >= length) {
vma = left;
continue;
}
}

gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0;
check_current:
/* Check if current node has a suitable gap */
if (gap_start > high_limit)
return -ENOMEM;
if (gap_end >= low_limit &&
gap_end > gap_start && gap_end - gap_start >= length)
goto found;

/* Visit right subtree if it looks promising */
if (vma->vm_rb.rb_right) {
struct vm_area_struct *right =
rb_entry(vma->vm_rb.rb_right,
struct vm_area_struct, vm_rb);
if (right->rb_subtree_gap >= length) {
vma = right;
continue;
}
}

/* Go back up the rbtree to find next candidate node */
while (true) {
struct rb_node *prev = &vma->vm_rb;
if (!rb_parent(prev))
goto check_highest;
vma = rb_entry(rb_parent(prev),
struct vm_area_struct, vm_rb);
if (prev == vma->vm_rb.rb_left) {
gap_start = vm_end_gap(vma->vm_prev);
gap_end = vm_start_gap(vma);
goto check_current;
}
}
}

check_highest:
/* Check highest gap, which does not precede any rbtree node */
gap_start = mm->highest_vm_end;
gap_end = ULONG_MAX; /* Only for VM_BUG_ON below */
if (gap_start > high_limit)
return -ENOMEM;

found:
/* We found a suitable gap. Clip it with the original low_limit. */
if (gap_start < info->low_limit)
gap_start = info->low_limit;

/* Adjust gap address to the desired alignment */
gap_start += (info->align_offset - gap_start) & info->align_mask;

VM_BUG_ON(gap_start + info->length > info->high_limit);
VM_BUG_ON(gap_start + info->length > gap_end);

VM_BUG_ON(gap != gap_start);
return gap_start;
gap = mas.index;
gap += (info->align_offset - gap) & info->align_mask;
return gap;
}

/**
* unmapped_area_topdown() - Find an area between the low_limit and the
* high_limit with * the correct alignment and offset at the highest available
* address, all from @info. Note: current->mm is used for the search.
*
* @info: The unmapped area information including the range (low_limit -
* hight_limit), the alignment offset and mask.
*
* Return: A memory address or -ENOMEM.
*/
static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma = NULL;
unsigned long length, low_limit, high_limit, gap_start, gap_end;
unsigned long gap;

MA_STATE(mas, &mm->mm_mt, 0, 0);
validate_mm_mt(mm);
unsigned long length, gap;

MA_STATE(mas, &current->mm->mm_mt, 0, 0);
/* Adjust search length to account for worst case alignment overhead */
length = info->length + info->align_mask;
if (length < info->length)
return -ENOMEM;

mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1,
length);
gap = mas.last + 1 - info->length;
gap -= (gap - info->align_offset) & info->align_mask;

/*
* Adjust search limits by the desired length.
* See implementation comment at top of unmapped_area().
*/
gap_end = info->high_limit;
if (gap_end < length)
return -ENOMEM;
high_limit = gap_end - length;

if (info->low_limit > high_limit)
if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1,
length))
return -ENOMEM;
low_limit = info->low_limit + length;

/* Check highest gap, which does not precede any rbtree node */
gap_start = mm->highest_vm_end;
if (gap_start <= high_limit)
goto found_highest;

/* Check if rbtree root looks promising */
if (RB_EMPTY_ROOT(&mm->mm_rb))
return -ENOMEM;
vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
if (vma->rb_subtree_gap < length)
return -ENOMEM;

while (true) {
/* Visit right subtree if it looks promising */
gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0;
if (gap_start <= high_limit && vma->vm_rb.rb_right) {
struct vm_area_struct *right =
rb_entry(vma->vm_rb.rb_right,
struct vm_area_struct, vm_rb);
if (right->rb_subtree_gap >= length) {
vma = right;
continue;
}
}

check_current:
/* Check if current node has a suitable gap */
gap_end = vm_start_gap(vma);
if (gap_end < low_limit)
return -ENOMEM;
if (gap_start <= high_limit &&
gap_end > gap_start && gap_end - gap_start >= length)
goto found;

/* Visit left subtree if it looks promising */
if (vma->vm_rb.rb_left) {
struct vm_area_struct *left =
rb_entry(vma->vm_rb.rb_left,
struct vm_area_struct, vm_rb);
if (left->rb_subtree_gap >= length) {
vma = left;
continue;
}
}

/* Go back up the rbtree to find next candidate node */
while (true) {
struct rb_node *prev = &vma->vm_rb;
if (!rb_parent(prev))
return -ENOMEM;
vma = rb_entry(rb_parent(prev),
struct vm_area_struct, vm_rb);
if (prev == vma->vm_rb.rb_right) {
gap_start = vma->vm_prev ?
vm_end_gap(vma->vm_prev) : 0;
goto check_current;
}
}
}

found:
/* We found a suitable gap. Clip it with the original high_limit. */
if (gap_end > info->high_limit)
gap_end = info->high_limit;

found_highest:
/* Compute highest gap address at the desired alignment */
gap_end -= info->length;
gap_end -= (gap_end - info->align_offset) & info->align_mask;

VM_BUG_ON(gap_end < info->low_limit);
VM_BUG_ON(gap_end < gap_start);

if (gap != gap_end) {
pr_err("%s: %p Gap was found: mt %lu gap_end %lu\n", __func__,
mm, gap, gap_end);
pr_err("window was %lu - %lu size %lu\n", info->high_limit,
info->low_limit, length);
pr_err("mas.min %lu max %lu mas.last %lu\n", mas.min, mas.max,
mas.last);
pr_err("mas.index %lu align mask %lu offset %lu\n", mas.index,
info->align_mask, info->align_offset);
pr_err("rb_find_vma find on %lu => %p (%p)\n", mas.index,
find_vma(mm, mas.index), vma);
#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
mt_dump(&mm->mm_mt);
#endif
{
struct vm_area_struct *dv = mm->mmap;

while (dv) {
pr_err("vma %p %lu-%lu\n", dv, dv->vm_start, dv->vm_end);
dv = dv->vm_next;
}
}
VM_BUG_ON(gap != gap_end);
}

return gap_end;
gap = mas.last + 1 - info->length;
gap -= (gap - info->align_offset) & info->align_mask;
return gap;
}

/*
Expand Down

0 comments on commit 3499a13

Please sign in to comment.