Skip to content

Commit

Permalink
Merge branch 'slab/next' into slab/for-linus
Browse files Browse the repository at this point in the history 
Fix up a trivial merge conflict with commit baaf1dd ("mm/slob: use
min_t() to compare ARCH_SLAB_MINALIGN") that did not go through the slab
tree.

Conflicts:
	mm/slob.c

Signed-off-by: Pekka Enberg <penberg@kernel.org>
  • Loading branch information
Pekka Enberg committed Dec 18, 2012
2 parents a304f83 + 4590685 commit 08afe22
Show file tree
Hide file tree
Showing 8 changed files with 236 additions and 333 deletions.
7 changes: 2 additions & 5 deletions include/linux/mm_types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,7 @@ struct page {
};

struct list_head list; /* slobs list of pages */
struct { /* slab fields */
struct kmem_cache *slab_cache;
struct slab *slab_page;
};
struct slab *slab_page; /* slab fields */
};

/* Remainder is not double word aligned */
Expand All @@ -146,7 +143,7 @@ struct page {
#if USE_SPLIT_PTLOCKS
spinlock_t ptl;
#endif
struct kmem_cache *slab; /* SLUB: Pointer to slab */
struct kmem_cache *slab_cache; /* SL[AU]B: Pointer to slab */
struct page *first_page; /* Compound tail pages */
};

Expand Down
9 changes: 8 additions & 1 deletion include/linux/slab.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,6 @@ struct kmem_cache *kmem_cache_create(const char *, size_t, size_t,
void kmem_cache_destroy(struct kmem_cache *);
int kmem_cache_shrink(struct kmem_cache *);
void kmem_cache_free(struct kmem_cache *, void *);
unsigned int kmem_cache_size(struct kmem_cache *);

/*
* Please use this macro to create slab caches. Simply specify the
Expand Down Expand Up @@ -388,6 +387,14 @@ static inline void *kzalloc_node(size_t size, gfp_t flags, int node)
return kmalloc_node(size, flags | __GFP_ZERO, node);
}

/*
* Determine the size of a slab object
*/
static inline unsigned int kmem_cache_size(struct kmem_cache *s)
{
return s->object_size;
}

void __init kmem_cache_init_late(void);

#endif /* _LINUX_SLAB_H */
6 changes: 5 additions & 1 deletion include/linux/slab_def.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,13 @@ struct kmem_cache {
* (see kmem_cache_init())
* We still use [NR_CPUS] and not [1] or [0] because cache_cache
* is statically defined, so we reserve the max number of cpus.
*
* We also need to guarantee that the list is able to accomodate a
* pointer for each node since "nodelists" uses the remainder of
* available pointers.
*/
struct kmem_list3 **nodelists;
struct array_cache *array[NR_CPUS];
struct array_cache *array[NR_CPUS + MAX_NUMNODES];
/*
* Do not add fields after array[]
*/
Expand Down
171 changes: 44 additions & 127 deletions mm/slab.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,23 +162,6 @@
*/
static bool pfmemalloc_active __read_mostly;

/* Legal flag mask for kmem_cache_create(). */
#if DEBUG
# define CREATE_MASK (SLAB_RED_ZONE | \
SLAB_POISON | SLAB_HWCACHE_ALIGN | \
SLAB_CACHE_DMA | \
SLAB_STORE_USER | \
SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
#else
# define CREATE_MASK (SLAB_HWCACHE_ALIGN | \
SLAB_CACHE_DMA | \
SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
#endif

/*
* kmem_bufctl_t:
*
Expand Down Expand Up @@ -564,15 +547,11 @@ static struct cache_names __initdata cache_names[] = {
#undef CACHE
};

static struct arraycache_init initarray_cache __initdata =
{ {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
static struct arraycache_init initarray_generic =
{ {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };

/* internal cache of cache description objs */
static struct kmem_list3 *kmem_cache_nodelists[MAX_NUMNODES];
static struct kmem_cache kmem_cache_boot = {
.nodelists = kmem_cache_nodelists,
.batchcount = 1,
.limit = BOOT_CPUCACHE_ENTRIES,
.shared = 1,
Expand Down Expand Up @@ -1576,29 +1555,34 @@ static void __init set_up_list3s(struct kmem_cache *cachep, int index)
}
}

/*
* The memory after the last cpu cache pointer is used for the
* the nodelists pointer.
*/
static void setup_nodelists_pointer(struct kmem_cache *cachep)
{
cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
}

/*
* Initialisation. Called after the page allocator have been initialised and
* before smp_init().
*/
void __init kmem_cache_init(void)
{
size_t left_over;
struct cache_sizes *sizes;
struct cache_names *names;
int i;
int order;
int node;

kmem_cache = &kmem_cache_boot;
setup_nodelists_pointer(kmem_cache);

if (num_possible_nodes() == 1)
use_alien_caches = 0;

for (i = 0; i < NUM_INIT_LISTS; i++) {
for (i = 0; i < NUM_INIT_LISTS; i++)
kmem_list3_init(&initkmem_list3[i]);
if (i < MAX_NUMNODES)
kmem_cache->nodelists[i] = NULL;
}

set_up_list3s(kmem_cache, CACHE_CACHE);

/*
Expand Down Expand Up @@ -1629,37 +1613,16 @@ void __init kmem_cache_init(void)
* 6) Resize the head arrays of the kmalloc caches to their final sizes.
*/

node = numa_mem_id();

/* 1) create the kmem_cache */
INIT_LIST_HEAD(&slab_caches);
list_add(&kmem_cache->list, &slab_caches);
kmem_cache->colour_off = cache_line_size();
kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
kmem_cache->nodelists[node] = &initkmem_list3[CACHE_CACHE + node];

/*
* struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
*/
kmem_cache->size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
nr_node_ids * sizeof(struct kmem_list3 *);
kmem_cache->object_size = kmem_cache->size;
kmem_cache->size = ALIGN(kmem_cache->object_size,
cache_line_size());
kmem_cache->reciprocal_buffer_size =
reciprocal_value(kmem_cache->size);

for (order = 0; order < MAX_ORDER; order++) {
cache_estimate(order, kmem_cache->size,
cache_line_size(), 0, &left_over, &kmem_cache->num);
if (kmem_cache->num)
break;
}
BUG_ON(!kmem_cache->num);
kmem_cache->gfporder = order;
kmem_cache->colour = left_over / kmem_cache->colour_off;
kmem_cache->slab_size = ALIGN(kmem_cache->num * sizeof(kmem_bufctl_t) +
sizeof(struct slab), cache_line_size());
create_boot_cache(kmem_cache, "kmem_cache",
offsetof(struct kmem_cache, array[nr_cpu_ids]) +
nr_node_ids * sizeof(struct kmem_list3 *),
SLAB_HWCACHE_ALIGN);
list_add(&kmem_cache->list, &slab_caches);

/* 2+3) create the kmalloc caches */
sizes = malloc_sizes;
Expand All @@ -1671,23 +1634,13 @@ void __init kmem_cache_init(void)
* bug.
*/

sizes[INDEX_AC].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
sizes[INDEX_AC].cs_cachep->name = names[INDEX_AC].name;
sizes[INDEX_AC].cs_cachep->size = sizes[INDEX_AC].cs_size;
sizes[INDEX_AC].cs_cachep->object_size = sizes[INDEX_AC].cs_size;
sizes[INDEX_AC].cs_cachep->align = ARCH_KMALLOC_MINALIGN;
__kmem_cache_create(sizes[INDEX_AC].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
list_add(&sizes[INDEX_AC].cs_cachep->list, &slab_caches);

if (INDEX_AC != INDEX_L3) {
sizes[INDEX_L3].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
sizes[INDEX_L3].cs_cachep->name = names[INDEX_L3].name;
sizes[INDEX_L3].cs_cachep->size = sizes[INDEX_L3].cs_size;
sizes[INDEX_L3].cs_cachep->object_size = sizes[INDEX_L3].cs_size;
sizes[INDEX_L3].cs_cachep->align = ARCH_KMALLOC_MINALIGN;
__kmem_cache_create(sizes[INDEX_L3].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
list_add(&sizes[INDEX_L3].cs_cachep->list, &slab_caches);
}
sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name,
sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS);

if (INDEX_AC != INDEX_L3)
sizes[INDEX_L3].cs_cachep =
create_kmalloc_cache(names[INDEX_L3].name,
sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS);

slab_early_init = 0;

Expand All @@ -1699,24 +1652,14 @@ void __init kmem_cache_init(void)
* Note for systems short on memory removing the alignment will
* allow tighter packing of the smaller caches.
*/
if (!sizes->cs_cachep) {
sizes->cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
sizes->cs_cachep->name = names->name;
sizes->cs_cachep->size = sizes->cs_size;
sizes->cs_cachep->object_size = sizes->cs_size;
sizes->cs_cachep->align = ARCH_KMALLOC_MINALIGN;
__kmem_cache_create(sizes->cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
list_add(&sizes->cs_cachep->list, &slab_caches);
}
if (!sizes->cs_cachep)
sizes->cs_cachep = create_kmalloc_cache(names->name,
sizes->cs_size, ARCH_KMALLOC_FLAGS);

#ifdef CONFIG_ZONE_DMA
sizes->cs_dmacachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
sizes->cs_dmacachep->name = names->name_dma;
sizes->cs_dmacachep->size = sizes->cs_size;
sizes->cs_dmacachep->object_size = sizes->cs_size;
sizes->cs_dmacachep->align = ARCH_KMALLOC_MINALIGN;
__kmem_cache_create(sizes->cs_dmacachep,
ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA| SLAB_PANIC);
list_add(&sizes->cs_dmacachep->list, &slab_caches);
sizes->cs_dmacachep = create_kmalloc_cache(
names->name_dma, sizes->cs_size,
SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
#endif
sizes++;
names++;
Expand All @@ -1727,7 +1670,6 @@ void __init kmem_cache_init(void)

ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);

BUG_ON(cpu_cache_get(kmem_cache) != &initarray_cache.cache);
memcpy(ptr, cpu_cache_get(kmem_cache),
sizeof(struct arraycache_init));
/*
Expand Down Expand Up @@ -2282,15 +2224,23 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)

if (slab_state == DOWN) {
/*
* Note: the first kmem_cache_create must create the cache
* Note: Creation of first cache (kmem_cache).
* The setup_list3s is taken care
* of by the caller of __kmem_cache_create
*/
cachep->array[smp_processor_id()] = &initarray_generic.cache;
slab_state = PARTIAL;
} else if (slab_state == PARTIAL) {
/*
* Note: the second kmem_cache_create must create the cache
* that's used by kmalloc(24), otherwise the creation of
* further caches will BUG().
*/
cachep->array[smp_processor_id()] = &initarray_generic.cache;

/*
* If the cache that's used by kmalloc(sizeof(kmem_list3)) is
* the first cache, then we need to set up all its list3s,
* the second cache, then we need to set up all its list3s,
* otherwise the creation of further caches will BUG().
*/
set_up_list3s(cachep, SIZE_AC);
Expand All @@ -2299,6 +2249,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
else
slab_state = PARTIAL_ARRAYCACHE;
} else {
/* Remaining boot caches */
cachep->array[smp_processor_id()] =
kmalloc(sizeof(struct arraycache_init), gfp);

Expand Down Expand Up @@ -2331,11 +2282,8 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)

/**
* __kmem_cache_create - Create a cache.
* @name: A string which is used in /proc/slabinfo to identify this cache.
* @size: The size of objects to be created in this cache.
* @align: The required alignment for the objects.
* @cachep: cache management descriptor
* @flags: SLAB flags
* @ctor: A constructor for the objects.
*
* Returns a ptr to the cache on success, NULL on failure.
* Cannot be called within a int, but can be interrupted.
Expand Down Expand Up @@ -2378,11 +2326,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
if (flags & SLAB_DESTROY_BY_RCU)
BUG_ON(flags & SLAB_POISON);
#endif
/*
* Always checks flags, a caller might be expecting debug support which
* isn't available.
*/
BUG_ON(flags & ~CREATE_MASK);

/*
* Check that size is in terms of words. This is needed to avoid
Expand All @@ -2394,22 +2337,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
size &= ~(BYTES_PER_WORD - 1);
}

/* calculate the final buffer alignment: */

/* 1) arch recommendation: can be overridden for debug */
if (flags & SLAB_HWCACHE_ALIGN) {
/*
* Default alignment: as specified by the arch code. Except if
* an object is really small, then squeeze multiple objects into
* one cacheline.
*/
ralign = cache_line_size();
while (size <= ralign / 2)
ralign /= 2;
} else {
ralign = BYTES_PER_WORD;
}

/*
* Redzoning and user store require word alignment or possibly larger.
* Note this will be overridden by architecture or caller mandated
Expand All @@ -2426,10 +2353,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
size &= ~(REDZONE_ALIGN - 1);
}

/* 2) arch mandated alignment */
if (ralign < ARCH_SLAB_MINALIGN) {
ralign = ARCH_SLAB_MINALIGN;
}
/* 3) caller mandated alignment */
if (ralign < cachep->align) {
ralign = cachep->align;
Expand All @@ -2447,7 +2370,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
else
gfp = GFP_NOWAIT;

cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
setup_nodelists_pointer(cachep);
#if DEBUG

/*
Expand Down Expand Up @@ -3969,12 +3892,6 @@ void kfree(const void *objp)
}
EXPORT_SYMBOL(kfree);

unsigned int kmem_cache_size(struct kmem_cache *cachep)
{
return cachep->object_size;
}
EXPORT_SYMBOL(kmem_cache_size);

/*
* This initializes kmem_list3 or resizes various caches for all nodes.
*/
Expand Down
Loading

0 comments on commit 08afe22

Please sign in to comment.