mm: nommu: sort mm->mmap list properly
When I was reading nommu code, I found that it handles the vma list/tree
in an unusual way. IIUC, because there can be more than one
identical/overrapped vmas in the list/tree, it sorts the tree more
strictly and does a linear search on the tree. But it doesn't applied to
the list (i.e. the list could be constructed in a different order than
the tree so that we can't use the list when finding the first vma in that
order).
Since inserting/sorting a vma in the tree and link is done at the same
time, we can easily construct both of them in the same order. And linear
searching on the tree could be more costly than doing it on the list, it
can be converted to use the list.
Also, after the commit 297c5eee3724 ("mm: make the vma list be doubly
linked") made the list be doubly linked, there were a couple of code need
to be fixed to construct the list properly.
Patch 1/6 is a preparation. It maintains the list sorted same as the tree
and construct doubly-linked list properly. Patch 2/6 is a simple
optimization for the vma deletion. Patch 3/6 and 4/6 convert tree
traversal to list traversal and the rest are simple fixes and cleanups.
This patch:
@vma added into @mm should be sorted by start addr, end addr and VMA
struct addr in that order because we may get identical VMAs in the @mm.
However this was true only for the rbtree, not for the list.
This patch fixes this by remembering 'rb_prev' during the tree traversal
like find_vma_prepare() does and linking the @vma via __vma_link_list().
After this patch, we can iterate the whole VMAs in correct order simply by
using @mm->mmap list.
[akpm@linux-foundation.org: avoid duplicating __vma_link_list()]
Signed-off-by: Namhyung Kim <namhyung@gmail.com>
Acked-by: Greg Ungerer <gerg@uclinux.org>
Cc: David Howells <dhowells@redhat.com>
Cc: Paul Mundt <lethal@linux-sh.org>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
diff --git a/mm/internal.h b/mm/internal.h
index 9d0ced8e..d071d38 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -66,6 +66,10 @@
return page_private(page);
}
+/* mm/util.c */
+void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
+ struct vm_area_struct *prev, struct rb_node *rb_parent);
+
#ifdef CONFIG_MMU
extern long mlock_vma_pages_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end);
diff --git a/mm/mmap.c b/mm/mmap.c
index 4fb5464..e76f8d7 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -398,29 +398,6 @@
return vma;
}
-static inline void
-__vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
- struct vm_area_struct *prev, struct rb_node *rb_parent)
-{
- struct vm_area_struct *next;
-
- vma->vm_prev = prev;
- if (prev) {
- next = prev->vm_next;
- prev->vm_next = vma;
- } else {
- mm->mmap = vma;
- if (rb_parent)
- next = rb_entry(rb_parent,
- struct vm_area_struct, vm_rb);
- else
- next = NULL;
- }
- vma->vm_next = next;
- if (next)
- next->vm_prev = vma;
-}
-
void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
struct rb_node **rb_link, struct rb_node *rb_parent)
{
diff --git a/mm/nommu.c b/mm/nommu.c
index 5afce48..0b16cb4 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -680,9 +680,9 @@
*/
static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
{
- struct vm_area_struct *pvma, **pp, *next;
+ struct vm_area_struct *pvma, *prev;
struct address_space *mapping;
- struct rb_node **p, *parent;
+ struct rb_node **p, *parent, *rb_prev;
kenter(",%p", vma);
@@ -703,7 +703,7 @@
}
/* add the VMA to the tree */
- parent = NULL;
+ parent = rb_prev = NULL;
p = &mm->mm_rb.rb_node;
while (*p) {
parent = *p;
@@ -713,17 +713,20 @@
* (the latter is necessary as we may get identical VMAs) */
if (vma->vm_start < pvma->vm_start)
p = &(*p)->rb_left;
- else if (vma->vm_start > pvma->vm_start)
+ else if (vma->vm_start > pvma->vm_start) {
+ rb_prev = parent;
p = &(*p)->rb_right;
- else if (vma->vm_end < pvma->vm_end)
+ } else if (vma->vm_end < pvma->vm_end)
p = &(*p)->rb_left;
- else if (vma->vm_end > pvma->vm_end)
+ else if (vma->vm_end > pvma->vm_end) {
+ rb_prev = parent;
p = &(*p)->rb_right;
- else if (vma < pvma)
+ } else if (vma < pvma)
p = &(*p)->rb_left;
- else if (vma > pvma)
+ else if (vma > pvma) {
+ rb_prev = parent;
p = &(*p)->rb_right;
- else
+ } else
BUG();
}
@@ -731,20 +734,11 @@
rb_insert_color(&vma->vm_rb, &mm->mm_rb);
/* add VMA to the VMA list also */
- for (pp = &mm->mmap; (pvma = *pp); pp = &(*pp)->vm_next) {
- if (pvma->vm_start > vma->vm_start)
- break;
- if (pvma->vm_start < vma->vm_start)
- continue;
- if (pvma->vm_end < vma->vm_end)
- break;
- }
+ prev = NULL;
+ if (rb_prev)
+ prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
- next = *pp;
- *pp = vma;
- vma->vm_next = next;
- if (next)
- next->vm_prev = vma;
+ __vma_link_list(mm, vma, prev, parent);
}
/*
diff --git a/mm/util.c b/mm/util.c
index e7b103a..88ea1bd 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -6,6 +6,8 @@
#include <linux/sched.h>
#include <asm/uaccess.h>
+#include "internal.h"
+
#define CREATE_TRACE_POINTS
#include <trace/events/kmem.h>
@@ -215,6 +217,28 @@
}
EXPORT_SYMBOL(strndup_user);
+void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
+ struct vm_area_struct *prev, struct rb_node *rb_parent)
+{
+ struct vm_area_struct *next;
+
+ vma->vm_prev = prev;
+ if (prev) {
+ next = prev->vm_next;
+ prev->vm_next = vma;
+ } else {
+ mm->mmap = vma;
+ if (rb_parent)
+ next = rb_entry(rb_parent,
+ struct vm_area_struct, vm_rb);
+ else
+ next = NULL;
+ }
+ vma->vm_next = next;
+ if (next)
+ next->vm_prev = vma;
+}
+
#if defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
void arch_pick_mmap_layout(struct mm_struct *mm)
{
