Skip to content

Commit

Permalink
memcg: enhance memcg iterator to support predicates
Browse files Browse the repository at this point in the history 
The caller of the iterator might know that some nodes or even subtrees
should be skipped but there is no way to tell iterators about that so the
only choice left is to let iterators to visit each node and do the
selection outside of the iterating code.  This, however, doesn't scale
well with hierarchies with many groups where only few groups are
interesting.

This patch adds mem_cgroup_iter_cond variant of the iterator with a
callback which gets called for every visited node.  There are three
possible ways how the callback can influence the walk.  Either the node is
visited, it is skipped but the tree walk continues down the tree or the
whole subtree of the current group is skipped.

[hughd@google.com: fix memcg-less page reclaim]
Signed-off-by: Michal Hocko <mhocko@suse.cz>
Cc: Balbir Singh <bsingharora@gmail.com>
Cc: Glauber Costa <glommer@openvz.org>
Cc: Greg Thelen <gthelen@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Michel Lespinasse <walken@google.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Ying Han <yinghan@google.com>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
  • Loading branch information
Michal Hocko authored and Linus Torvalds committed Sep 12, 2013
1 parent a5b7c87 commit de57780
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 32 deletions.
49 changes: 43 additions & 6 deletions include/linux/memcontrol.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,23 @@ struct mem_cgroup_reclaim_cookie {
unsigned int generation;
};

enum mem_cgroup_filter_t {
VISIT, /* visit current node */
SKIP, /* skip the current node and continue traversal */
SKIP_TREE, /* skip the whole subtree and continue traversal */
};

/*
* mem_cgroup_filter_t predicate might instruct mem_cgroup_iter_cond how to
* iterate through the hierarchy tree. Each tree element is checked by the
* predicate before it is returned by the iterator. If a filter returns
* SKIP or SKIP_TREE then the iterator code continues traversal (with the
* next node down the hierarchy or the next node that doesn't belong under the
* memcg's subtree).
*/
typedef enum mem_cgroup_filter_t
(*mem_cgroup_iter_filter)(struct mem_cgroup *memcg, struct mem_cgroup *root);

#ifdef CONFIG_MEMCG
/*
* All "charge" functions with gfp_mask should use GFP_KERNEL or
Expand Down Expand Up @@ -108,9 +125,18 @@ mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
extern void mem_cgroup_end_migration(struct mem_cgroup *memcg,
struct page *oldpage, struct page *newpage, bool migration_ok);

struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
struct mem_cgroup *,
struct mem_cgroup_reclaim_cookie *);
struct mem_cgroup *mem_cgroup_iter_cond(struct mem_cgroup *root,
struct mem_cgroup *prev,
struct mem_cgroup_reclaim_cookie *reclaim,
mem_cgroup_iter_filter cond);

static inline struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
struct mem_cgroup *prev,
struct mem_cgroup_reclaim_cookie *reclaim)
{
return mem_cgroup_iter_cond(root, prev, reclaim, NULL);
}

void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);

/*
Expand Down Expand Up @@ -180,7 +206,8 @@ static inline void mem_cgroup_dec_page_stat(struct page *page,
mem_cgroup_update_page_stat(page, idx, -1);
}

bool mem_cgroup_soft_reclaim_eligible(struct mem_cgroup *memcg,
enum mem_cgroup_filter_t
mem_cgroup_soft_reclaim_eligible(struct mem_cgroup *memcg,
struct mem_cgroup *root);

void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx);
Expand Down Expand Up @@ -295,6 +322,15 @@ static inline void mem_cgroup_end_migration(struct mem_cgroup *memcg,
struct page *oldpage, struct page *newpage, bool migration_ok)
{
}
static inline struct mem_cgroup *
mem_cgroup_iter_cond(struct mem_cgroup *root,
struct mem_cgroup *prev,
struct mem_cgroup_reclaim_cookie *reclaim,
mem_cgroup_iter_filter cond)
{
/* first call must return non-NULL, second return NULL */
return (struct mem_cgroup *)(unsigned long)!prev;
}

static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup *root,
Expand Down Expand Up @@ -358,10 +394,11 @@ static inline void mem_cgroup_dec_page_stat(struct page *page,
}

static inline
bool mem_cgroup_soft_reclaim_eligible(struct mem_cgroup *memcg,
enum mem_cgroup_filter_t
mem_cgroup_soft_reclaim_eligible(struct mem_cgroup *memcg,
struct mem_cgroup *root)
{
return false;
return VISIT;
}

static inline void mem_cgroup_split_huge_fixup(struct page *head)
Expand Down
70 changes: 55 additions & 15 deletions mm/memcontrol.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -875,14 +875,23 @@ struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
return memcg;
}

static enum mem_cgroup_filter_t
mem_cgroup_filter(struct mem_cgroup *memcg, struct mem_cgroup *root,
mem_cgroup_iter_filter cond)
{
if (!cond)
return VISIT;
return cond(memcg, root);
}

/*
* Returns a next (in a pre-order walk) alive memcg (with elevated css
* ref. count) or NULL if the whole root's subtree has been visited.
*
* helper function to be used by mem_cgroup_iter
*/
static struct mem_cgroup *__mem_cgroup_iter_next(struct mem_cgroup *root,
struct mem_cgroup *last_visited)
struct mem_cgroup *last_visited, mem_cgroup_iter_filter cond)
{
struct cgroup_subsys_state *prev_css, *next_css;

Expand All @@ -900,11 +909,31 @@ static struct mem_cgroup *__mem_cgroup_iter_next(struct mem_cgroup *root,
if (next_css) {
struct mem_cgroup *mem = mem_cgroup_from_css(next_css);

if (css_tryget(&mem->css))
return mem;
else {
switch (mem_cgroup_filter(mem, root, cond)) {
case SKIP:
prev_css = next_css;
goto skip_node;
case SKIP_TREE:
if (mem == root)
return NULL;
/*
* css_rightmost_descendant is not an optimal way to
* skip through a subtree (especially for imbalanced
* trees leaning to right) but that's what we have right
* now. More effective solution would be traversing
* right-up for first non-NULL without calling
* css_next_descendant_pre afterwards.
*/
prev_css = css_rightmost_descendant(next_css);
goto skip_node;
case VISIT:
if (css_tryget(&mem->css))
return mem;
else {
prev_css = next_css;
goto skip_node;
}
break;
}
}

Expand Down Expand Up @@ -968,6 +997,7 @@ static void mem_cgroup_iter_update(struct mem_cgroup_reclaim_iter *iter,
* @root: hierarchy root
* @prev: previously returned memcg, NULL on first invocation
* @reclaim: cookie for shared reclaim walks, NULL for full walks
* @cond: filter for visited nodes, NULL for no filter
*
* Returns references to children of the hierarchy below @root, or
* @root itself, or %NULL after a full round-trip.
Expand All @@ -980,15 +1010,18 @@ static void mem_cgroup_iter_update(struct mem_cgroup_reclaim_iter *iter,
* divide up the memcgs in the hierarchy among all concurrent
* reclaimers operating on the same zone and priority.
*/
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
struct mem_cgroup *mem_cgroup_iter_cond(struct mem_cgroup *root,
struct mem_cgroup *prev,
struct mem_cgroup_reclaim_cookie *reclaim)
struct mem_cgroup_reclaim_cookie *reclaim,
mem_cgroup_iter_filter cond)
{
struct mem_cgroup *memcg = NULL;
struct mem_cgroup *last_visited = NULL;

if (mem_cgroup_disabled())
return NULL;
if (mem_cgroup_disabled()) {
/* first call must return non-NULL, second return NULL */
return (struct mem_cgroup *)(unsigned long)!prev;
}

if (!root)
root = root_mem_cgroup;
Expand All @@ -999,7 +1032,9 @@ struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
if (!root->use_hierarchy && root != root_mem_cgroup) {
if (prev)
goto out_css_put;
return root;
if (mem_cgroup_filter(root, root, cond) == VISIT)
return root;
return NULL;
}

rcu_read_lock();
Expand All @@ -1022,7 +1057,7 @@ struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
last_visited = mem_cgroup_iter_load(iter, root, &seq);
}

memcg = __mem_cgroup_iter_next(root, last_visited);
memcg = __mem_cgroup_iter_next(root, last_visited, cond);

if (reclaim) {
mem_cgroup_iter_update(iter, last_visited, memcg, seq);
Expand All @@ -1033,7 +1068,11 @@ struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
reclaim->generation = iter->generation;
}

if (prev && !memcg)
/*
* We have finished the whole tree walk or no group has been
* visited because filter told us to skip the root node.
*/
if (!memcg && (prev || (cond && !last_visited)))
goto out_unlock;
}
out_unlock:
Expand Down Expand Up @@ -1778,26 +1817,27 @@ int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
* a) it is over its soft limit
* b) any parent up the hierarchy is over its soft limit
*/
bool mem_cgroup_soft_reclaim_eligible(struct mem_cgroup *memcg,
enum mem_cgroup_filter_t
mem_cgroup_soft_reclaim_eligible(struct mem_cgroup *memcg,
struct mem_cgroup *root)
{
struct mem_cgroup *parent = memcg;

if (res_counter_soft_limit_excess(&memcg->res))
return true;
return VISIT;

/*
* If any parent up to the root in the hierarchy is over its soft limit
* then we have to obey and reclaim from this group as well.
*/
while((parent = parent_mem_cgroup(parent))) {
if (res_counter_soft_limit_excess(&parent->res))
return true;
return VISIT;
if (parent == root)
break;
}

return false;
return SKIP;
}

/*
Expand Down
16 changes: 5 additions & 11 deletions mm/vmscan.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2151,21 +2151,16 @@ __shrink_zone(struct zone *zone, struct scan_control *sc, bool soft_reclaim)
.zone = zone,
.priority = sc->priority,
};
struct mem_cgroup *memcg;
struct mem_cgroup *memcg = NULL;
mem_cgroup_iter_filter filter = (soft_reclaim) ?
mem_cgroup_soft_reclaim_eligible : NULL;

nr_reclaimed = sc->nr_reclaimed;
nr_scanned = sc->nr_scanned;

memcg = mem_cgroup_iter(root, NULL, &reclaim);
do {
while ((memcg = mem_cgroup_iter_cond(root, memcg, &reclaim, filter))) {
struct lruvec *lruvec;

if (soft_reclaim &&
!mem_cgroup_soft_reclaim_eligible(memcg, root)) {
memcg = mem_cgroup_iter(root, memcg, &reclaim);
continue;
}

lruvec = mem_cgroup_zone_lruvec(zone, memcg);

shrink_lruvec(lruvec, sc);
Expand All @@ -2185,8 +2180,7 @@ __shrink_zone(struct zone *zone, struct scan_control *sc, bool soft_reclaim)
mem_cgroup_iter_break(root, memcg);
break;
}
memcg = mem_cgroup_iter(root, memcg, &reclaim);
} while (memcg);
}

vmpressure(sc->gfp_mask, sc->target_mem_cgroup,
sc->nr_scanned - nr_scanned,
Expand Down

0 comments on commit de57780

Please sign in to comment.