Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history
yaml
---
r: 213238
b: refs/heads/master
c: bbddff0
h: refs/heads/master
v: v3
  • Loading branch information
Tejun Heo committed Sep 8, 2010
1 parent 62fd235 commit 84df364
Show file tree
Hide file tree
Showing 7 changed files with 72 additions and 66 deletions.
2 changes: 1 addition & 1 deletion [refs]
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: 6abad5acac09921f4944af77d3860f82d49f528d
refs/heads/master: bbddff0545878a8649c091a9dd7c43ce91516734
29 changes: 5 additions & 24 deletions trunk/include/linux/percpu.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@
preempt_enable(); \
} while (0)

#ifdef CONFIG_SMP

/* minimum unit size, also is the maximum supported allocation size */
#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10)

Expand Down Expand Up @@ -137,37 +135,20 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size,
* dynamically allocated. Non-atomic access to the current CPU's
* version should probably be combined with get_cpu()/put_cpu().
*/
#ifdef CONFIG_SMP
#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
#else
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); })
#endif

extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
extern bool is_kernel_percpu_address(unsigned long addr);

#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
#if !defined(CONFIG_SMP) || !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA)
extern void __init setup_per_cpu_areas(void);
#endif
extern void __init percpu_init_late(void);

#else /* CONFIG_SMP */

#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); })

/* can't distinguish from other static vars, always false */
static inline bool is_kernel_percpu_address(unsigned long addr)
{
return false;
}

static inline void __init setup_per_cpu_areas(void) { }

static inline void __init percpu_init_late(void) { }

static inline void *pcpu_lpage_remapped(void *kaddr)
{
return NULL;
}

#endif /* CONFIG_SMP */

extern void __percpu *__alloc_percpu(size_t size, size_t align);
extern void free_percpu(void __percpu *__pdata);
extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
Expand Down
8 changes: 8 additions & 0 deletions trunk/mm/Kconfig
Original file line number Diff line number Diff line change
Expand Up @@ -301,3 +301,11 @@ config NOMMU_INITIAL_TRIM_EXCESS
of 1 says that all excess pages should be trimmed.

See Documentation/nommu-mmap.txt for more information.

#
# UP and nommu archs use km based percpu allocator
#
config NEED_PER_CPU_KM
depends on !SMP
bool
default y
7 changes: 1 addition & 6 deletions trunk/mm/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ obj-y := bootmem.o filemap.o mempool.o oom_kill.o fadvise.o \
maccess.o page_alloc.o page-writeback.o \
readahead.o swap.o truncate.o vmscan.o shmem.o \
prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \
page_isolation.o mm_init.o mmu_context.o \
page_isolation.o mm_init.o mmu_context.o percpu.o \
$(mmu-y)
obj-y += init-mm.o

Expand All @@ -36,11 +36,6 @@ obj-$(CONFIG_FAILSLAB) += failslab.o
obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o
obj-$(CONFIG_FS_XIP) += filemap_xip.o
obj-$(CONFIG_MIGRATION) += migrate.o
ifdef CONFIG_SMP
obj-y += percpu.o
else
obj-y += percpu_up.o
endif
obj-$(CONFIG_QUICKLIST) += quicklist.o
obj-$(CONFIG_CGROUP_MEM_RES_CTLR) += memcontrol.o page_cgroup.o
obj-$(CONFIG_MEMORY_FAILURE) += memory-failure.o
Expand Down
2 changes: 1 addition & 1 deletion trunk/mm/percpu-km.c
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
* chunk size is not aligned. percpu-km code will whine about it.
*/

#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
#if defined(CONFIG_SMP) && defined(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK)
#error "contiguous percpu allocation is incompatible with paged first chunk"
#endif

Expand Down
60 changes: 56 additions & 4 deletions trunk/mm/percpu.c
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
#define PCPU_SLOT_BASE_SHIFT 5 /* 1-31 shares the same slot */
#define PCPU_DFL_MAP_ALLOC 16 /* start a map with 16 ents */

#ifdef CONFIG_SMP
/* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */
#ifndef __addr_to_pcpu_ptr
#define __addr_to_pcpu_ptr(addr) \
Expand All @@ -89,6 +90,11 @@
(unsigned long)pcpu_base_addr - \
(unsigned long)__per_cpu_start)
#endif
#else /* CONFIG_SMP */
/* on UP, it's always identity mapped */
#define __addr_to_pcpu_ptr(addr) (void __percpu *)(addr)
#define __pcpu_ptr_to_addr(ptr) (void __force *)(ptr)
#endif /* CONFIG_SMP */

struct pcpu_chunk {
struct list_head list; /* linked to pcpu_slot lists */
Expand Down Expand Up @@ -949,6 +955,7 @@ EXPORT_SYMBOL_GPL(free_percpu);
*/
bool is_kernel_percpu_address(unsigned long addr)
{
#ifdef CONFIG_SMP
const size_t static_size = __per_cpu_end - __per_cpu_start;
void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
unsigned int cpu;
Expand All @@ -959,6 +966,8 @@ bool is_kernel_percpu_address(unsigned long addr)
if ((void *)addr >= start && (void *)addr < start + static_size)
return true;
}
#endif
/* on UP, can't distinguish from other static vars, always false */
return false;
}

Expand Down Expand Up @@ -1066,6 +1075,8 @@ void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai)
free_bootmem(__pa(ai), ai->__ai_size);
}

#if defined(CONFIG_SMP) && (defined(CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK) || \
defined(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK))
/**
* pcpu_build_alloc_info - build alloc_info considering distances between CPUs
* @reserved_size: the size of reserved percpu area in bytes
Expand Down Expand Up @@ -1220,6 +1231,8 @@ static struct pcpu_alloc_info * __init pcpu_build_alloc_info(

return ai;
}
#endif /* CONFIG_SMP && (CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK ||
CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) */

/**
* pcpu_dump_alloc_info - print out information about pcpu_alloc_info
Expand Down Expand Up @@ -1363,7 +1376,9 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,

/* sanity checks */
PCPU_SETUP_BUG_ON(ai->nr_groups <= 0);
#ifdef CONFIG_SMP
PCPU_SETUP_BUG_ON(!ai->static_size);
#endif
PCPU_SETUP_BUG_ON(!base_addr);
PCPU_SETUP_BUG_ON(ai->unit_size < size_sum);
PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK);
Expand Down Expand Up @@ -1488,6 +1503,8 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
return 0;
}

#ifdef CONFIG_SMP

const char *pcpu_fc_names[PCPU_FC_NR] __initdata = {
[PCPU_FC_AUTO] = "auto",
[PCPU_FC_EMBED] = "embed",
Expand Down Expand Up @@ -1758,8 +1775,9 @@ int __init pcpu_page_first_chunk(size_t reserved_size,
}
#endif /* CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK */

#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
/*
* Generic percpu area setup.
* Generic SMP percpu area setup.
*
* The embedding helper is used because its behavior closely resembles
* the original non-dynamic generic percpu area setup. This is
Expand All @@ -1770,7 +1788,6 @@ int __init pcpu_page_first_chunk(size_t reserved_size,
* on the physical linear memory mapping which uses large page
* mappings on applicable archs.
*/
#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(__per_cpu_offset);

Expand Down Expand Up @@ -1799,13 +1816,48 @@ void __init setup_per_cpu_areas(void)
PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, NULL,
pcpu_dfl_fc_alloc, pcpu_dfl_fc_free);
if (rc < 0)
panic("Failed to initialized percpu areas.");
panic("Failed to initialize percpu areas.");

delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
for_each_possible_cpu(cpu)
__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
}
#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */
#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */

#else /* CONFIG_SMP */

/*
* UP percpu area setup.
*
* UP always uses km-based percpu allocator with identity mapping.
* Static percpu variables are indistinguishable from the usual static
* variables and don't require any special preparation.
*/
void __init setup_per_cpu_areas(void)
{
const size_t unit_size =
roundup_pow_of_two(max_t(size_t, PCPU_MIN_UNIT_SIZE,
PERCPU_DYNAMIC_RESERVE));
struct pcpu_alloc_info *ai;
void *fc;

ai = pcpu_alloc_alloc_info(1, 1);
fc = __alloc_bootmem(unit_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
if (!ai || !fc)
panic("Failed to allocate memory for percpu areas.");

ai->dyn_size = unit_size;
ai->unit_size = unit_size;
ai->atom_size = unit_size;
ai->alloc_size = unit_size;
ai->groups[0].nr_units = 1;
ai->groups[0].cpu_map[0] = 0;

if (pcpu_setup_first_chunk(ai, fc) < 0)
panic("Failed to initialize percpu areas.");
}

#endif /* CONFIG_SMP */

/*
* First and reserved chunks are initialized with temporary allocation
Expand Down
30 changes: 0 additions & 30 deletions trunk/mm/percpu_up.c

This file was deleted.

0 comments on commit 84df364

Please sign in to comment.