Skip to content

Commit

Permalink
[PATCH] mm: page_state opt
Browse files Browse the repository at this point in the history 
Optimise page_state manipulations by introducing interrupt unsafe accessors
to page_state fields.  Callers must provide their own locking (either
disable interrupts or not update from interrupt context).

Switch over the hot callsites that can easily be moved under interrupts off
sections.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
  • Loading branch information
Nick Piggin authored and Linus Torvalds committed Jan 6, 2006
1 parent d3cb487 commit a74609f
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 65 deletions.
43 changes: 32 additions & 11 deletions include/linux/page-flags.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,22 +144,33 @@ struct page_state {
extern void get_page_state(struct page_state *ret);
extern void get_page_state_node(struct page_state *ret, int node);
extern void get_full_page_state(struct page_state *ret);
extern unsigned long __read_page_state(unsigned long offset);
extern void __mod_page_state(unsigned long offset, unsigned long delta);
extern unsigned long read_page_state_offset(unsigned long offset);
extern void mod_page_state_offset(unsigned long offset, unsigned long delta);
extern void __mod_page_state_offset(unsigned long offset, unsigned long delta);

#define read_page_state(member) \
__read_page_state(offsetof(struct page_state, member))
read_page_state_offset(offsetof(struct page_state, member))

#define mod_page_state(member, delta) \
__mod_page_state(offsetof(struct page_state, member), (delta))
mod_page_state_offset(offsetof(struct page_state, member), (delta))

#define inc_page_state(member) mod_page_state(member, 1UL)
#define dec_page_state(member) mod_page_state(member, 0UL - 1)
#define add_page_state(member,delta) mod_page_state(member, (delta))
#define sub_page_state(member,delta) mod_page_state(member, 0UL - (delta))
#define __mod_page_state(member, delta) \
__mod_page_state_offset(offsetof(struct page_state, member), (delta))

#define mod_page_state_zone(zone, member, delta) \
do { \
#define inc_page_state(member) mod_page_state(member, 1UL)
#define dec_page_state(member) mod_page_state(member, 0UL - 1)
#define add_page_state(member,delta) mod_page_state(member, (delta))
#define sub_page_state(member,delta) mod_page_state(member, 0UL - (delta))

#define __inc_page_state(member) __mod_page_state(member, 1UL)
#define __dec_page_state(member) __mod_page_state(member, 0UL - 1)
#define __add_page_state(member,delta) __mod_page_state(member, (delta))
#define __sub_page_state(member,delta) __mod_page_state(member, 0UL - (delta))

#define page_state(member) (*__page_state(offsetof(struct page_state, member)))

#define state_zone_offset(zone, member) \
({ \
unsigned offset; \
if (is_highmem(zone)) \
offset = offsetof(struct page_state, member##_high); \
Expand All @@ -169,7 +180,17 @@ extern void __mod_page_state(unsigned long offset, unsigned long delta);
offset = offsetof(struct page_state, member##_dma32); \
else \
offset = offsetof(struct page_state, member##_dma); \
__mod_page_state(offset, (delta)); \
offset; \
})

#define __mod_page_state_zone(zone, member, delta) \
do { \
__mod_page_state_offset(state_zone_offset(zone, member), (delta)); \
} while (0)

#define mod_page_state_zone(zone, member, delta) \
do { \
mod_page_state_offset(state_zone_offset(zone, member), (delta)); \
} while (0)

/*
Expand Down
89 changes: 50 additions & 39 deletions mm/page_alloc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,9 +424,9 @@ void __free_pages_ok(struct page *page, unsigned int order)
return;

list_add(&page->lru, &list);
mod_page_state(pgfree, 1 << order);
kernel_map_pages(page, 1<<order, 0);
local_irq_save(flags);
__mod_page_state(pgfree, 1 << order);
free_pages_bulk(page_zone(page), 1, &list, order);
local_irq_restore(flags);
}
Expand Down Expand Up @@ -674,18 +674,14 @@ void drain_local_pages(void)
}
#endif /* CONFIG_PM */

static void zone_statistics(struct zonelist *zonelist, struct zone *z)
static void zone_statistics(struct zonelist *zonelist, struct zone *z, int cpu)
{
#ifdef CONFIG_NUMA
unsigned long flags;
int cpu;
pg_data_t *pg = z->zone_pgdat;
pg_data_t *orig = zonelist->zones[0]->zone_pgdat;
struct per_cpu_pageset *p;

local_irq_save(flags);
cpu = smp_processor_id();
p = zone_pcp(z,cpu);
p = zone_pcp(z, cpu);
if (pg == orig) {
p->numa_hit++;
} else {
Expand All @@ -696,7 +692,6 @@ static void zone_statistics(struct zonelist *zonelist, struct zone *z)
p->local_node++;
else
p->other_node++;
local_irq_restore(flags);
#endif
}

Expand All @@ -716,11 +711,11 @@ static void fastcall free_hot_cold_page(struct page *page, int cold)
if (free_pages_check(page))
return;

inc_page_state(pgfree);
kernel_map_pages(page, 1, 0);

pcp = &zone_pcp(zone, get_cpu())->pcp[cold];
local_irq_save(flags);
__inc_page_state(pgfree);
list_add(&page->lru, &pcp->list);
pcp->count++;
if (pcp->count >= pcp->high)
Expand Down Expand Up @@ -753,49 +748,58 @@ static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
* we cheat by calling it from here, in the order > 0 path. Saves a branch
* or two.
*/
static struct page *
buffered_rmqueue(struct zone *zone, int order, gfp_t gfp_flags)
static struct page *buffered_rmqueue(struct zonelist *zonelist,
struct zone *zone, int order, gfp_t gfp_flags)
{
unsigned long flags;
struct page *page;
int cold = !!(gfp_flags & __GFP_COLD);
int cpu;

again:
cpu = get_cpu();
if (order == 0) {
struct per_cpu_pages *pcp;

page = NULL;
pcp = &zone_pcp(zone, get_cpu())->pcp[cold];
pcp = &zone_pcp(zone, cpu)->pcp[cold];
local_irq_save(flags);
if (!pcp->count)
if (!pcp->count) {
pcp->count += rmqueue_bulk(zone, 0,
pcp->batch, &pcp->list);
if (likely(pcp->count)) {
page = list_entry(pcp->list.next, struct page, lru);
list_del(&page->lru);
pcp->count--;
if (unlikely(!pcp->count))
goto failed;
}
local_irq_restore(flags);
put_cpu();
page = list_entry(pcp->list.next, struct page, lru);
list_del(&page->lru);
pcp->count--;
} else {
spin_lock_irqsave(&zone->lock, flags);
page = __rmqueue(zone, order);
spin_unlock_irqrestore(&zone->lock, flags);
spin_unlock(&zone->lock);
if (!page)
goto failed;
}

if (page != NULL) {
BUG_ON(bad_range(zone, page));
mod_page_state_zone(zone, pgalloc, 1 << order);
if (prep_new_page(page, order))
goto again;
__mod_page_state_zone(zone, pgalloc, 1 << order);
zone_statistics(zonelist, zone, cpu);
local_irq_restore(flags);
put_cpu();

if (gfp_flags & __GFP_ZERO)
prep_zero_page(page, order, gfp_flags);
BUG_ON(bad_range(zone, page));
if (prep_new_page(page, order))
goto again;

if (order && (gfp_flags & __GFP_COMP))
prep_compound_page(page, order);
}
if (gfp_flags & __GFP_ZERO)
prep_zero_page(page, order, gfp_flags);

if (order && (gfp_flags & __GFP_COMP))
prep_compound_page(page, order);
return page;

failed:
local_irq_restore(flags);
put_cpu();
return NULL;
}

#define ALLOC_NO_WATERMARKS 0x01 /* don't check watermarks at all */
Expand Down Expand Up @@ -871,9 +875,8 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order,
continue;
}

page = buffered_rmqueue(*z, order, gfp_mask);
page = buffered_rmqueue(zonelist, *z, order, gfp_mask);
if (page) {
zone_statistics(zonelist, *z);
break;
}
} while (*(++z) != NULL);
Expand Down Expand Up @@ -1248,7 +1251,7 @@ void get_full_page_state(struct page_state *ret)
__get_page_state(ret, sizeof(*ret) / sizeof(unsigned long), &mask);
}

unsigned long __read_page_state(unsigned long offset)
unsigned long read_page_state_offset(unsigned long offset)
{
unsigned long ret = 0;
int cpu;
Expand All @@ -1262,18 +1265,26 @@ unsigned long __read_page_state(unsigned long offset)
return ret;
}

void __mod_page_state(unsigned long offset, unsigned long delta)
void __mod_page_state_offset(unsigned long offset, unsigned long delta)
{
void *ptr;

ptr = &__get_cpu_var(page_states);
*(unsigned long *)(ptr + offset) += delta;
}
EXPORT_SYMBOL(__mod_page_state_offset);

void mod_page_state_offset(unsigned long offset, unsigned long delta)
{
unsigned long flags;
void* ptr;
void *ptr;

local_irq_save(flags);
ptr = &__get_cpu_var(page_states);
*(unsigned long*)(ptr + offset) += delta;
*(unsigned long *)(ptr + offset) += delta;
local_irq_restore(flags);
}

EXPORT_SYMBOL(__mod_page_state);
EXPORT_SYMBOL(mod_page_state_offset);

void __get_zone_counts(unsigned long *active, unsigned long *inactive,
unsigned long *free, struct pglist_data *pgdat)
Expand Down
10 changes: 7 additions & 3 deletions mm/rmap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,11 @@ static void __page_set_anon_rmap(struct page *page,

page->index = linear_page_index(vma, address);

inc_page_state(nr_mapped);
/*
* nr_mapped state can be updated without turning off
* interrupts because it is not modified via interrupt.
*/
__inc_page_state(nr_mapped);
}

/**
Expand Down Expand Up @@ -498,7 +502,7 @@ void page_add_file_rmap(struct page *page)
BUG_ON(!pfn_valid(page_to_pfn(page)));

if (atomic_inc_and_test(&page->_mapcount))
inc_page_state(nr_mapped);
__inc_page_state(nr_mapped);
}

/**
Expand All @@ -522,7 +526,7 @@ void page_remove_rmap(struct page *page)
*/
if (page_test_and_clear_dirty(page))
set_page_dirty(page);
dec_page_state(nr_mapped);
__dec_page_state(nr_mapped);
}
}

Expand Down
27 changes: 15 additions & 12 deletions mm/vmscan.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -645,16 +645,17 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc)
goto done;

max_scan -= nr_scan;
if (current_is_kswapd())
mod_page_state_zone(zone, pgscan_kswapd, nr_scan);
else
mod_page_state_zone(zone, pgscan_direct, nr_scan);
nr_freed = shrink_list(&page_list, sc);
if (current_is_kswapd())
mod_page_state(kswapd_steal, nr_freed);
mod_page_state_zone(zone, pgsteal, nr_freed);

spin_lock_irq(&zone->lru_lock);
local_irq_disable();
if (current_is_kswapd()) {
__mod_page_state_zone(zone, pgscan_kswapd, nr_scan);
__mod_page_state(kswapd_steal, nr_freed);
} else
__mod_page_state_zone(zone, pgscan_direct, nr_scan);
__mod_page_state_zone(zone, pgsteal, nr_freed);

spin_lock(&zone->lru_lock);
/*
* Put back any unfreeable pages.
*/
Expand Down Expand Up @@ -816,11 +817,13 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc)
}
}
zone->nr_active += pgmoved;
spin_unlock_irq(&zone->lru_lock);
pagevec_release(&pvec);
spin_unlock(&zone->lru_lock);

__mod_page_state_zone(zone, pgrefill, pgscanned);
__mod_page_state(pgdeactivate, pgdeactivate);
local_irq_enable();

mod_page_state_zone(zone, pgrefill, pgscanned);
mod_page_state(pgdeactivate, pgdeactivate);
pagevec_release(&pvec);
}

/*
Expand Down

0 comments on commit a74609f

Please sign in to comment.