Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history 
yaml
---
r: 16338
b: refs/heads/master
c: a74609f
h: refs/heads/master
v: v3
  • Loading branch information
Nick Piggin authored and Linus Torvalds committed Jan 6, 2006
1 parent c3522bd commit 21948b6
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 66 deletions.
2 changes: 1 addition & 1 deletion [refs]
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: d3cb487149bd706aa6aeb02042332a450978dc1c
refs/heads/master: a74609fafa2e5cc31d558012abaaa55ec9ad9da4
43 changes: 32 additions & 11 deletions trunk/include/linux/page-flags.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,22 +144,33 @@ struct page_state {
extern void get_page_state(struct page_state *ret);
extern void get_page_state_node(struct page_state *ret, int node);
extern void get_full_page_state(struct page_state *ret);
extern unsigned long __read_page_state(unsigned long offset);
extern void __mod_page_state(unsigned long offset, unsigned long delta);
extern unsigned long read_page_state_offset(unsigned long offset);
extern void mod_page_state_offset(unsigned long offset, unsigned long delta);
extern void __mod_page_state_offset(unsigned long offset, unsigned long delta);

#define read_page_state(member) \
__read_page_state(offsetof(struct page_state, member))
read_page_state_offset(offsetof(struct page_state, member))

#define mod_page_state(member, delta) \
__mod_page_state(offsetof(struct page_state, member), (delta))
mod_page_state_offset(offsetof(struct page_state, member), (delta))

#define inc_page_state(member) mod_page_state(member, 1UL)
#define dec_page_state(member) mod_page_state(member, 0UL - 1)
#define add_page_state(member,delta) mod_page_state(member, (delta))
#define sub_page_state(member,delta) mod_page_state(member, 0UL - (delta))
#define __mod_page_state(member, delta) \
__mod_page_state_offset(offsetof(struct page_state, member), (delta))

#define mod_page_state_zone(zone, member, delta) \
do { \
#define inc_page_state(member) mod_page_state(member, 1UL)
#define dec_page_state(member) mod_page_state(member, 0UL - 1)
#define add_page_state(member,delta) mod_page_state(member, (delta))
#define sub_page_state(member,delta) mod_page_state(member, 0UL - (delta))

#define __inc_page_state(member) __mod_page_state(member, 1UL)
#define __dec_page_state(member) __mod_page_state(member, 0UL - 1)
#define __add_page_state(member,delta) __mod_page_state(member, (delta))
#define __sub_page_state(member,delta) __mod_page_state(member, 0UL - (delta))

#define page_state(member) (*__page_state(offsetof(struct page_state, member)))

#define state_zone_offset(zone, member) \
({ \
unsigned offset; \
if (is_highmem(zone)) \
offset = offsetof(struct page_state, member##_high); \
Expand All @@ -169,7 +180,17 @@ extern void __mod_page_state(unsigned long offset, unsigned long delta);
offset = offsetof(struct page_state, member##_dma32); \
else \
offset = offsetof(struct page_state, member##_dma); \
__mod_page_state(offset, (delta)); \
offset; \
})

#define __mod_page_state_zone(zone, member, delta) \
do { \
__mod_page_state_offset(state_zone_offset(zone, member), (delta)); \
} while (0)

#define mod_page_state_zone(zone, member, delta) \
do { \
mod_page_state_offset(state_zone_offset(zone, member), (delta)); \
} while (0)

/*
Expand Down
89 changes: 50 additions & 39 deletions trunk/mm/page_alloc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,9 +424,9 @@ void __free_pages_ok(struct page *page, unsigned int order)
return;

list_add(&page->lru, &list);
mod_page_state(pgfree, 1 << order);
kernel_map_pages(page, 1<<order, 0);
local_irq_save(flags);
__mod_page_state(pgfree, 1 << order);
free_pages_bulk(page_zone(page), 1, &list, order);
local_irq_restore(flags);
}
Expand Down Expand Up @@ -674,18 +674,14 @@ void drain_local_pages(void)
}
#endif /* CONFIG_PM */

static void zone_statistics(struct zonelist *zonelist, struct zone *z)
static void zone_statistics(struct zonelist *zonelist, struct zone *z, int cpu)
{
#ifdef CONFIG_NUMA
unsigned long flags;
int cpu;
pg_data_t *pg = z->zone_pgdat;
pg_data_t *orig = zonelist->zones[0]->zone_pgdat;
struct per_cpu_pageset *p;

local_irq_save(flags);
cpu = smp_processor_id();
p = zone_pcp(z,cpu);
p = zone_pcp(z, cpu);
if (pg == orig) {
p->numa_hit++;
} else {
Expand All @@ -696,7 +692,6 @@ static void zone_statistics(struct zonelist *zonelist, struct zone *z)
p->local_node++;
else
p->other_node++;
local_irq_restore(flags);
#endif
}

Expand All @@ -716,11 +711,11 @@ static void fastcall free_hot_cold_page(struct page *page, int cold)
if (free_pages_check(page))
return;

inc_page_state(pgfree);
kernel_map_pages(page, 1, 0);

pcp = &zone_pcp(zone, get_cpu())->pcp[cold];
local_irq_save(flags);
__inc_page_state(pgfree);
list_add(&page->lru, &pcp->list);
pcp->count++;
if (pcp->count >= pcp->high)
Expand Down Expand Up @@ -753,49 +748,58 @@ static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
* we cheat by calling it from here, in the order > 0 path. Saves a branch
* or two.
*/
static struct page *
buffered_rmqueue(struct zone *zone, int order, gfp_t gfp_flags)
static struct page *buffered_rmqueue(struct zonelist *zonelist,
struct zone *zone, int order, gfp_t gfp_flags)
{
unsigned long flags;
struct page *page;
int cold = !!(gfp_flags & __GFP_COLD);
int cpu;

again:
cpu = get_cpu();
if (order == 0) {
struct per_cpu_pages *pcp;

page = NULL;
pcp = &zone_pcp(zone, get_cpu())->pcp[cold];
pcp = &zone_pcp(zone, cpu)->pcp[cold];
local_irq_save(flags);
if (!pcp->count)
if (!pcp->count) {
pcp->count += rmqueue_bulk(zone, 0,
pcp->batch, &pcp->list);
if (likely(pcp->count)) {
page = list_entry(pcp->list.next, struct page, lru);
list_del(&page->lru);
pcp->count--;
if (unlikely(!pcp->count))
goto failed;
}
local_irq_restore(flags);
put_cpu();
page = list_entry(pcp->list.next, struct page, lru);
list_del(&page->lru);
pcp->count--;
} else {
spin_lock_irqsave(&zone->lock, flags);
page = __rmqueue(zone, order);
spin_unlock_irqrestore(&zone->lock, flags);
spin_unlock(&zone->lock);
if (!page)
goto failed;
}

if (page != NULL) {
BUG_ON(bad_range(zone, page));
mod_page_state_zone(zone, pgalloc, 1 << order);
if (prep_new_page(page, order))
goto again;
__mod_page_state_zone(zone, pgalloc, 1 << order);
zone_statistics(zonelist, zone, cpu);
local_irq_restore(flags);
put_cpu();

if (gfp_flags & __GFP_ZERO)
prep_zero_page(page, order, gfp_flags);
BUG_ON(bad_range(zone, page));
if (prep_new_page(page, order))
goto again;

if (order && (gfp_flags & __GFP_COMP))
prep_compound_page(page, order);
}
if (gfp_flags & __GFP_ZERO)
prep_zero_page(page, order, gfp_flags);

if (order && (gfp_flags & __GFP_COMP))
prep_compound_page(page, order);
return page;

failed:
local_irq_restore(flags);
put_cpu();
return NULL;
}

#define ALLOC_NO_WATERMARKS 0x01 /* don't check watermarks at all */
Expand Down Expand Up @@ -871,9 +875,8 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order,
continue;
}

page = buffered_rmqueue(*z, order, gfp_mask);
page = buffered_rmqueue(zonelist, *z, order, gfp_mask);
if (page) {
zone_statistics(zonelist, *z);
break;
}
} while (*(++z) != NULL);
Expand Down Expand Up @@ -1248,7 +1251,7 @@ void get_full_page_state(struct page_state *ret)
__get_page_state(ret, sizeof(*ret) / sizeof(unsigned long), &mask);
}

unsigned long __read_page_state(unsigned long offset)
unsigned long read_page_state_offset(unsigned long offset)
{
unsigned long ret = 0;
int cpu;
Expand All @@ -1262,18 +1265,26 @@ unsigned long __read_page_state(unsigned long offset)
return ret;
}

void __mod_page_state(unsigned long offset, unsigned long delta)
void __mod_page_state_offset(unsigned long offset, unsigned long delta)
{
void *ptr;

ptr = &__get_cpu_var(page_states);
*(unsigned long *)(ptr + offset) += delta;
}
EXPORT_SYMBOL(__mod_page_state_offset);

void mod_page_state_offset(unsigned long offset, unsigned long delta)
{
unsigned long flags;
void* ptr;
void *ptr;

local_irq_save(flags);
ptr = &__get_cpu_var(page_states);
*(unsigned long*)(ptr + offset) += delta;
*(unsigned long *)(ptr + offset) += delta;
local_irq_restore(flags);
}

EXPORT_SYMBOL(__mod_page_state);
EXPORT_SYMBOL(mod_page_state_offset);

void __get_zone_counts(unsigned long *active, unsigned long *inactive,
unsigned long *free, struct pglist_data *pgdat)
Expand Down
10 changes: 7 additions & 3 deletions trunk/mm/rmap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,11 @@ static void __page_set_anon_rmap(struct page *page,

page->index = linear_page_index(vma, address);

inc_page_state(nr_mapped);
/*
* nr_mapped state can be updated without turning off
* interrupts because it is not modified via interrupt.
*/
__inc_page_state(nr_mapped);
}

/**
Expand Down Expand Up @@ -498,7 +502,7 @@ void page_add_file_rmap(struct page *page)
BUG_ON(!pfn_valid(page_to_pfn(page)));

if (atomic_inc_and_test(&page->_mapcount))
inc_page_state(nr_mapped);
__inc_page_state(nr_mapped);
}

/**
Expand All @@ -522,7 +526,7 @@ void page_remove_rmap(struct page *page)
*/
if (page_test_and_clear_dirty(page))
set_page_dirty(page);
dec_page_state(nr_mapped);
__dec_page_state(nr_mapped);
}
}

Expand Down
27 changes: 15 additions & 12 deletions trunk/mm/vmscan.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -645,16 +645,17 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc)
goto done;

max_scan -= nr_scan;
if (current_is_kswapd())
mod_page_state_zone(zone, pgscan_kswapd, nr_scan);
else
mod_page_state_zone(zone, pgscan_direct, nr_scan);
nr_freed = shrink_list(&page_list, sc);
if (current_is_kswapd())
mod_page_state(kswapd_steal, nr_freed);
mod_page_state_zone(zone, pgsteal, nr_freed);

spin_lock_irq(&zone->lru_lock);
local_irq_disable();
if (current_is_kswapd()) {
__mod_page_state_zone(zone, pgscan_kswapd, nr_scan);
__mod_page_state(kswapd_steal, nr_freed);
} else
__mod_page_state_zone(zone, pgscan_direct, nr_scan);
__mod_page_state_zone(zone, pgsteal, nr_freed);

spin_lock(&zone->lru_lock);
/*
* Put back any unfreeable pages.
*/
Expand Down Expand Up @@ -816,11 +817,13 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc)
}
}
zone->nr_active += pgmoved;
spin_unlock_irq(&zone->lru_lock);
pagevec_release(&pvec);
spin_unlock(&zone->lru_lock);

__mod_page_state_zone(zone, pgrefill, pgscanned);
__mod_page_state(pgdeactivate, pgdeactivate);
local_irq_enable();

mod_page_state_zone(zone, pgrefill, pgscanned);
mod_page_state(pgdeactivate, pgdeactivate);
pagevec_release(&pvec);
}

/*
Expand Down

0 comments on commit 21948b6

Please sign in to comment.