Skip to content

Commit

Permalink
lib/stackdepot: replace CONFIG_STACK_HASH_ORDER with automatic sizing
Browse files Browse the repository at this point in the history
As Linus explained [1], setting the stackdepot hash table size as a config
option is suboptimal, especially as stackdepot becomes a dependency of
less "expert" subsystems than initially (e.g.  DRM, networking,
SLUB_DEBUG):

: (a) it introduces a new compile-time question that isn't sane to ask
: a regular user, but is now exposed to regular users.

: (b) this by default uses 1MB of memory for a feature that didn't in
: the past, so now if you have small machines you need to make sure you
: make a special kernel config for them.

Ideally we would employ rhashtable for fully automatic resizing, which
should be feasible for many of the new users, but problematic for the
original users with restricted context that call __stack_depot_save() with
can_alloc == false, i.e.  KASAN.

However we can easily remove the config option and scale the hash table
automatically with system memory.  The STACK_HASH_MASK constant becomes
stack_hash_mask variable and is used only in one mask operation, so the
overhead should be negligible to none.  For early allocation we can employ
the existing alloc_large_system_hash() function and perform similar
scaling for the late allocation.

The existing limits of the config option (between 4k and 1M buckets) are
preserved, and scaling factor is set to one bucket per 16kB memory so on
64bit the max 1M buckets (8MB memory) is achieved with 16GB system, while
a 1GB system will use 512kB.

Because KASAN is reported to need the maximum number of buckets even with
smaller amounts of memory [2], set it as such when kasan_enabled().

If needed, the automatic scaling could be complemented with a boot-time
kernel parameter, but it feels pointless to add it without a specific use
case.

[1] https://lore.kernel.org/all/CAHk-=wjC5nS+fnf6EzRD9yQRJApAhxx7gRB87ZV+pAWo9oVrTg@mail.gmail.com/
[2] https://lore.kernel.org/all/CACT4Y+Y4GZfXOru2z5tFPzFdaSUd+GFc6KVL=bsa0+1m197cQQ@mail.gmail.com/

Link: https://lkml.kernel.org/r/20220620150249.16814-1-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reported-by: Linus Torvalds <torvalds@linux-foundation.org>
Acked-by: Dmitry Vyukov <dvyukov@google.com>
Cc: Marco Elver <elver@google.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Konovalov <andreyknvl@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  • Loading branch information
Vlastimil Babka authored and akpm committed Jul 18, 2022
1 parent 62df90b commit f998792
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 19 deletions.
9 changes: 0 additions & 9 deletions lib/Kconfig
Original file line number Diff line number Diff line change
Expand Up @@ -685,15 +685,6 @@ config STACKDEPOT_ALWAYS_INIT
bool
select STACKDEPOT

config STACK_HASH_ORDER
int "stack depot hash size (12 => 4KB, 20 => 1024KB)"
range 12 20
default 20
depends on STACKDEPOT
help
Select the hash size as a power of 2 for the stackdepot hash table.
Choose a lower value to reduce the memory impact.

config REF_TRACKER
bool
depends on STACKTRACE_SUPPORT
Expand Down
59 changes: 49 additions & 10 deletions lib/stackdepot.c
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include <linux/string.h>
#include <linux/types.h>
#include <linux/memblock.h>
#include <linux/kasan-enabled.h>

#define DEPOT_STACK_BITS (sizeof(depot_stack_handle_t) * 8)

Expand Down Expand Up @@ -145,10 +146,16 @@ depot_alloc_stack(unsigned long *entries, int size, u32 hash, void **prealloc)
return stack;
}

#define STACK_HASH_SIZE (1L << CONFIG_STACK_HASH_ORDER)
#define STACK_HASH_MASK (STACK_HASH_SIZE - 1)
/* one hash table bucket entry per 16kB of memory */
#define STACK_HASH_SCALE 14
/* limited between 4k and 1M buckets */
#define STACK_HASH_ORDER_MIN 12
#define STACK_HASH_ORDER_MAX 20
#define STACK_HASH_SEED 0x9747b28c

static unsigned int stack_hash_order;
static unsigned int stack_hash_mask;

static bool stack_depot_disable;
static struct stack_record **stack_table;

Expand All @@ -175,21 +182,31 @@ void __init stack_depot_want_early_init(void)

int __init stack_depot_early_init(void)
{
size_t size;
unsigned long entries = 0;

/* This is supposed to be called only once, from mm_init() */
if (WARN_ON(__stack_depot_early_init_passed))
return 0;

__stack_depot_early_init_passed = true;

if (kasan_enabled() && !stack_hash_order)
stack_hash_order = STACK_HASH_ORDER_MAX;

if (!__stack_depot_want_early_init || stack_depot_disable)
return 0;

size = (STACK_HASH_SIZE * sizeof(struct stack_record *));
pr_info("Stack Depot early init allocating hash table with memblock_alloc, %zu bytes\n",
size);
stack_table = memblock_alloc(size, SMP_CACHE_BYTES);
if (stack_hash_order)
entries = 1UL << stack_hash_order;
stack_table = alloc_large_system_hash("stackdepot",
sizeof(struct stack_record *),
entries,
STACK_HASH_SCALE,
HASH_EARLY | HASH_ZERO,
NULL,
&stack_hash_mask,
1UL << STACK_HASH_ORDER_MIN,
1UL << STACK_HASH_ORDER_MAX);

if (!stack_table) {
pr_err("Stack Depot hash table allocation failed, disabling\n");
Expand All @@ -207,13 +224,35 @@ int stack_depot_init(void)

mutex_lock(&stack_depot_init_mutex);
if (!stack_depot_disable && !stack_table) {
pr_info("Stack Depot allocating hash table with kvcalloc\n");
stack_table = kvcalloc(STACK_HASH_SIZE, sizeof(struct stack_record *), GFP_KERNEL);
unsigned long entries;
int scale = STACK_HASH_SCALE;

if (stack_hash_order) {
entries = 1UL << stack_hash_order;
} else {
entries = nr_free_buffer_pages();
entries = roundup_pow_of_two(entries);

if (scale > PAGE_SHIFT)
entries >>= (scale - PAGE_SHIFT);
else
entries <<= (PAGE_SHIFT - scale);
}

if (entries < 1UL << STACK_HASH_ORDER_MIN)
entries = 1UL << STACK_HASH_ORDER_MIN;
if (entries > 1UL << STACK_HASH_ORDER_MAX)
entries = 1UL << STACK_HASH_ORDER_MAX;

pr_info("Stack Depot allocating hash table of %lu entries with kvcalloc\n",
entries);
stack_table = kvcalloc(entries, sizeof(struct stack_record *), GFP_KERNEL);
if (!stack_table) {
pr_err("Stack Depot hash table allocation failed, disabling\n");
stack_depot_disable = true;
ret = -ENOMEM;
}
stack_hash_mask = entries - 1;
}
mutex_unlock(&stack_depot_init_mutex);
return ret;
Expand Down Expand Up @@ -386,7 +425,7 @@ depot_stack_handle_t __stack_depot_save(unsigned long *entries,
goto fast_exit;

hash = hash_stack(entries, nr_entries);
bucket = &stack_table[hash & STACK_HASH_MASK];
bucket = &stack_table[hash & stack_hash_mask];

/*
* Fast path: look the stack trace up without locking.
Expand Down

0 comments on commit f998792

Please sign in to comment.