From 46234253b9363894a254844a6550b4cc5f3edfe8 Mon Sep 17 00:00:00 2001 From: Hannes Frederic Sowa Date: Thu, 8 Oct 2015 01:20:35 +0200 Subject: [PATCH 1/5] net: move net_get_random_once to lib There's no good reason why users outside of networking should not be using this facility, f.e. for initializing their seeds. Therefore, make it accessible from there as get_random_once(). Signed-off-by: Hannes Frederic Sowa Signed-off-by: Daniel Borkmann Acked-by: Alexei Starovoitov Signed-off-by: David S. Miller --- include/linux/net.h | 21 ++++------------- include/linux/once.h | 24 ++++++++++++++++++++ lib/Makefile | 3 ++- lib/once.c | 54 ++++++++++++++++++++++++++++++++++++++++++++ net/core/utils.c | 49 ---------------------------------------- 5 files changed, 84 insertions(+), 67 deletions(-) create mode 100644 include/linux/once.h create mode 100644 lib/once.c diff --git a/include/linux/net.h b/include/linux/net.h index 049d4b03c4c4d..70ac5e28e6b73 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -24,7 +24,8 @@ #include /* For O_CLOEXEC and O_NONBLOCK */ #include #include -#include +#include + #include struct poll_table_struct; @@ -250,22 +251,8 @@ do { \ } while (0) #endif -bool __net_get_random_once(void *buf, int nbytes, bool *done, - struct static_key *done_key); - -#define net_get_random_once(buf, nbytes) \ - ({ \ - bool ___ret = false; \ - static bool ___done = false; \ - static struct static_key ___once_key = \ - STATIC_KEY_INIT_TRUE; \ - if (static_key_true(&___once_key)) \ - ___ret = __net_get_random_once(buf, \ - nbytes, \ - &___done, \ - &___once_key); \ - ___ret; \ - }) +#define net_get_random_once(buf, nbytes) \ + get_random_once((buf), (nbytes)) int kernel_sendmsg(struct socket *sock, struct msghdr *msg, struct kvec *vec, size_t num, size_t len); diff --git a/include/linux/once.h b/include/linux/once.h new file mode 100644 index 0000000000000..2a83b538dd6ac --- /dev/null +++ b/include/linux/once.h @@ -0,0 +1,24 @@ +#ifndef _LINUX_ONCE_H +#define _LINUX_ONCE_H + +#include +#include + +bool __get_random_once(void *buf, int nbytes, bool *done, + struct static_key *once_key); + +#define get_random_once(buf, nbytes) \ + ({ \ + bool ___ret = false; \ + static bool ___done = false; \ + static struct static_key ___once_key = \ + STATIC_KEY_INIT_TRUE; \ + if (static_key_true(&___once_key)) \ + ___ret = __get_random_once((buf), \ + (nbytes), \ + &___done, \ + &___once_key); \ + ___ret; \ + }) + +#endif /* _LINUX_ONCE_H */ diff --git a/lib/Makefile b/lib/Makefile index 13a7c6ae3feca..8de3b012eac77 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -26,7 +26,8 @@ obj-y += bcd.o div64.o sort.o parser.o halfmd4.o debug_locks.o random32.o \ bust_spinlocks.o kasprintf.o bitmap.o scatterlist.o \ gcd.o lcm.o list_sort.o uuid.o flex_array.o iov_iter.o clz_ctz.o \ bsearch.o find_bit.o llist.o memweight.o kfifo.o \ - percpu-refcount.o percpu_ida.o rhashtable.o reciprocal_div.o + percpu-refcount.o percpu_ida.o rhashtable.o reciprocal_div.o \ + once.o obj-y += string_helpers.o obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o obj-y += hexdump.o diff --git a/lib/once.c b/lib/once.c new file mode 100644 index 0000000000000..2d5a7de17abaf --- /dev/null +++ b/lib/once.c @@ -0,0 +1,54 @@ +#include +#include +#include +#include + +struct __random_once_work { + struct work_struct work; + struct static_key *key; +}; + +static void __random_once_deferred(struct work_struct *w) +{ + struct __random_once_work *work; + + work = container_of(w, struct __random_once_work, work); + BUG_ON(!static_key_enabled(work->key)); + static_key_slow_dec(work->key); + kfree(work); +} + +static void __random_once_disable_jump(struct static_key *key) +{ + struct __random_once_work *w; + + w = kmalloc(sizeof(*w), GFP_ATOMIC); + if (!w) + return; + + INIT_WORK(&w->work, __random_once_deferred); + w->key = key; + schedule_work(&w->work); +} + +bool __get_random_once(void *buf, int nbytes, bool *done, + struct static_key *once_key) +{ + static DEFINE_SPINLOCK(lock); + unsigned long flags; + + spin_lock_irqsave(&lock, flags); + if (*done) { + spin_unlock_irqrestore(&lock, flags); + return false; + } + + get_random_bytes(buf, nbytes); + *done = true; + spin_unlock_irqrestore(&lock, flags); + + __random_once_disable_jump(once_key); + + return true; +} +EXPORT_SYMBOL(__get_random_once); diff --git a/net/core/utils.c b/net/core/utils.c index 3dffce953c39f..3d17ca8b47441 100644 --- a/net/core/utils.c +++ b/net/core/utils.c @@ -348,52 +348,3 @@ void inet_proto_csum_replace_by_diff(__sum16 *sum, struct sk_buff *skb, } } EXPORT_SYMBOL(inet_proto_csum_replace_by_diff); - -struct __net_random_once_work { - struct work_struct work; - struct static_key *key; -}; - -static void __net_random_once_deferred(struct work_struct *w) -{ - struct __net_random_once_work *work = - container_of(w, struct __net_random_once_work, work); - BUG_ON(!static_key_enabled(work->key)); - static_key_slow_dec(work->key); - kfree(work); -} - -static void __net_random_once_disable_jump(struct static_key *key) -{ - struct __net_random_once_work *w; - - w = kmalloc(sizeof(*w), GFP_ATOMIC); - if (!w) - return; - - INIT_WORK(&w->work, __net_random_once_deferred); - w->key = key; - schedule_work(&w->work); -} - -bool __net_get_random_once(void *buf, int nbytes, bool *done, - struct static_key *once_key) -{ - static DEFINE_SPINLOCK(lock); - unsigned long flags; - - spin_lock_irqsave(&lock, flags); - if (*done) { - spin_unlock_irqrestore(&lock, flags); - return false; - } - - get_random_bytes(buf, nbytes); - *done = true; - spin_unlock_irqrestore(&lock, flags); - - __net_random_once_disable_jump(once_key); - - return true; -} -EXPORT_SYMBOL(__net_get_random_once); From c90aeb948222a7b3d3391d232ec4f50fd8322ad3 Mon Sep 17 00:00:00 2001 From: Hannes Frederic Sowa Date: Thu, 8 Oct 2015 01:20:36 +0200 Subject: [PATCH 2/5] once: make helper generic for calling functions once Make the get_random_once() helper generic enough, so that functions in general would only be called once, where one user of this is then net_get_random_once(). The only implementation specific call is to get_random_bytes(), all the rest of this *_once() facility would be duplicated among different subsystems otherwise. The new DO_ONCE() helper will be used by prandom() later on, but might also be useful for other scenarios/subsystems as well where a one-time initialization in often-called, possibly fast path code could occur. Signed-off-by: Hannes Frederic Sowa Signed-off-by: Daniel Borkmann Acked-by: Alexei Starovoitov Signed-off-by: David S. Miller --- include/linux/once.h | 61 ++++++++++++++++++++++++++++++++++---------- lib/once.c | 50 +++++++++++++++++++++--------------- 2 files changed, 76 insertions(+), 35 deletions(-) diff --git a/include/linux/once.h b/include/linux/once.h index 2a83b538dd6ac..285f12cb40e6a 100644 --- a/include/linux/once.h +++ b/include/linux/once.h @@ -4,21 +4,54 @@ #include #include -bool __get_random_once(void *buf, int nbytes, bool *done, - struct static_key *once_key); +bool __do_once_start(bool *done, unsigned long *flags); +void __do_once_done(bool *done, struct static_key *once_key, + unsigned long *flags); -#define get_random_once(buf, nbytes) \ - ({ \ - bool ___ret = false; \ - static bool ___done = false; \ - static struct static_key ___once_key = \ - STATIC_KEY_INIT_TRUE; \ - if (static_key_true(&___once_key)) \ - ___ret = __get_random_once((buf), \ - (nbytes), \ - &___done, \ - &___once_key); \ - ___ret; \ +/* Call a function exactly once. The idea of DO_ONCE() is to perform + * a function call such as initialization of random seeds, etc, only + * once, where DO_ONCE() can live in the fast-path. After @func has + * been called with the passed arguments, the static key will patch + * out the condition into a nop. DO_ONCE() guarantees type safety of + * arguments! + * + * Not that the following is not equivalent ... + * + * DO_ONCE(func, arg); + * DO_ONCE(func, arg); + * + * ... to this version: + * + * void foo(void) + * { + * DO_ONCE(func, arg); + * } + * + * foo(); + * foo(); + * + * In case the one-time invocation could be triggered from multiple + * places, then a common helper function must be defined, so that only + * a single static key will be placed there! + */ +#define DO_ONCE(func, ...) \ + ({ \ + bool ___ret = false; \ + static bool ___done = false; \ + static struct static_key ___once_key = STATIC_KEY_INIT_TRUE; \ + if (static_key_true(&___once_key)) { \ + unsigned long ___flags; \ + ___ret = __do_once_start(&___done, &___flags); \ + if (unlikely(___ret)) { \ + func(__VA_ARGS__); \ + __do_once_done(&___done, &___once_key, \ + &___flags); \ + } \ + } \ + ___ret; \ }) +#define get_random_once(buf, nbytes) \ + DO_ONCE(get_random_bytes, (buf), (nbytes)) + #endif /* _LINUX_ONCE_H */ diff --git a/lib/once.c b/lib/once.c index 2d5a7de17abaf..05c8604627eb0 100644 --- a/lib/once.c +++ b/lib/once.c @@ -3,52 +3,60 @@ #include #include -struct __random_once_work { +struct once_work { struct work_struct work; struct static_key *key; }; -static void __random_once_deferred(struct work_struct *w) +static void once_deferred(struct work_struct *w) { - struct __random_once_work *work; + struct once_work *work; - work = container_of(w, struct __random_once_work, work); + work = container_of(w, struct once_work, work); BUG_ON(!static_key_enabled(work->key)); static_key_slow_dec(work->key); kfree(work); } -static void __random_once_disable_jump(struct static_key *key) +static void once_disable_jump(struct static_key *key) { - struct __random_once_work *w; + struct once_work *w; w = kmalloc(sizeof(*w), GFP_ATOMIC); if (!w) return; - INIT_WORK(&w->work, __random_once_deferred); + INIT_WORK(&w->work, once_deferred); w->key = key; schedule_work(&w->work); } -bool __get_random_once(void *buf, int nbytes, bool *done, - struct static_key *once_key) -{ - static DEFINE_SPINLOCK(lock); - unsigned long flags; +static DEFINE_SPINLOCK(once_lock); - spin_lock_irqsave(&lock, flags); +bool __do_once_start(bool *done, unsigned long *flags) + __acquires(once_lock) +{ + spin_lock_irqsave(&once_lock, *flags); if (*done) { - spin_unlock_irqrestore(&lock, flags); + spin_unlock_irqrestore(&once_lock, *flags); + /* Keep sparse happy by restoring an even lock count on + * this lock. In case we return here, we don't call into + * __do_once_done but return early in the DO_ONCE() macro. + */ + __acquire(once_lock); return false; } - get_random_bytes(buf, nbytes); - *done = true; - spin_unlock_irqrestore(&lock, flags); - - __random_once_disable_jump(once_key); - return true; } -EXPORT_SYMBOL(__get_random_once); +EXPORT_SYMBOL(__do_once_start); + +void __do_once_done(bool *done, struct static_key *once_key, + unsigned long *flags) + __releases(once_lock) +{ + *done = true; + spin_unlock_irqrestore(&once_lock, *flags); + once_disable_jump(once_key); +} +EXPORT_SYMBOL(__do_once_done); From 0dd50d1b0c003ab4f17597fe1198bb57a2fadc06 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 8 Oct 2015 01:20:37 +0200 Subject: [PATCH 3/5] random32: add prandom_seed_full_state helper Factor out the full reseed handling code that populates the state through get_random_bytes() and runs prandom_warmup(). The resulting prandom_seed_full_state() will be used later on in more than the current __prandom_reseed() user. Fix also two minor whitespace issues along the way. Signed-off-by: Daniel Borkmann Acked-by: Hannes Frederic Sowa Acked-by: Alexei Starovoitov Signed-off-by: David S. Miller --- lib/random32.c | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/lib/random32.c b/lib/random32.c index 0bee183fa18fa..36c09fb3fec90 100644 --- a/lib/random32.c +++ b/lib/random32.c @@ -181,7 +181,7 @@ void prandom_seed(u32 entropy) * No locking on the CPUs, but then somewhat random results are, well, * expected. */ - for_each_possible_cpu (i) { + for_each_possible_cpu(i) { struct rnd_state *state = &per_cpu(net_rand_state, i); state->s1 = __seed(state->s1 ^ entropy, 2U); @@ -201,7 +201,7 @@ static int __init prandom_init(void) prandom_state_selftest(); for_each_possible_cpu(i) { - struct rnd_state *state = &per_cpu(net_rand_state,i); + struct rnd_state *state = &per_cpu(net_rand_state, i); u32 weak_seed = (i + jiffies) ^ random_get_entropy(); prandom_seed_early(state, weak_seed, true); @@ -238,13 +238,30 @@ static void __init __prandom_start_seed_timer(void) add_timer(&seed_timer); } +static void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) +{ + int i; + + for_each_possible_cpu(i) { + struct rnd_state *state = per_cpu_ptr(pcpu_state, i); + u32 seeds[4]; + + get_random_bytes(&seeds, sizeof(seeds)); + state->s1 = __seed(seeds[0], 2U); + state->s2 = __seed(seeds[1], 8U); + state->s3 = __seed(seeds[2], 16U); + state->s4 = __seed(seeds[3], 128U); + + prandom_warmup(state); + } +} + /* * Generate better values after random number generator * is fully initialized. */ static void __prandom_reseed(bool late) { - int i; unsigned long flags; static bool latch = false; static DEFINE_SPINLOCK(lock); @@ -266,19 +283,7 @@ static void __prandom_reseed(bool late) goto out; latch = true; - - for_each_possible_cpu(i) { - struct rnd_state *state = &per_cpu(net_rand_state,i); - u32 seeds[4]; - - get_random_bytes(&seeds, sizeof(seeds)); - state->s1 = __seed(seeds[0], 2U); - state->s2 = __seed(seeds[1], 8U); - state->s3 = __seed(seeds[2], 16U); - state->s4 = __seed(seeds[3], 128U); - - prandom_warmup(state); - } + prandom_seed_full_state(&net_rand_state); out: spin_unlock_irqrestore(&lock, flags); } From 897ece56e714a2cc64e6914cb89a362d7021b36e Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 8 Oct 2015 01:20:38 +0200 Subject: [PATCH 4/5] random32: add prandom_init_once helper for own rngs Add a prandom_init_once() facility that works on the rnd_state, so that users that are keeping their own state independent from prandom_u32() can initialize their taus113 per cpu states. The motivation here is similar to net_get_random_once(): initialize the state as late as possible in the hope that enough entropy has been collected for the seeding. prandom_init_once() makes use of the recently introduced prandom_seed_full_state() helper and is generic enough so that it could also be used on fast-paths due to the DO_ONCE(). Signed-off-by: Daniel Borkmann Acked-by: Hannes Frederic Sowa Acked-by: Alexei Starovoitov Signed-off-by: David S. Miller --- include/linux/random.h | 6 ++++++ lib/random32.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/linux/random.h b/include/linux/random.h index e651874df2c9d..a75840c1aa714 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -7,6 +7,8 @@ #define _LINUX_RANDOM_H #include +#include + #include struct random_ready_callback { @@ -45,6 +47,10 @@ struct rnd_state { u32 prandom_u32_state(struct rnd_state *state); void prandom_bytes_state(struct rnd_state *state, void *buf, size_t nbytes); +void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state); + +#define prandom_init_once(pcpu_state) \ + DO_ONCE(prandom_seed_full_state, (pcpu_state)) /** * prandom_u32_max - returns a pseudo-random number in interval [0, ep_ro) diff --git a/lib/random32.c b/lib/random32.c index 36c09fb3fec90..12111910ccd07 100644 --- a/lib/random32.c +++ b/lib/random32.c @@ -238,7 +238,7 @@ static void __init __prandom_start_seed_timer(void) add_timer(&seed_timer); } -static void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) +void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) { int i; From 3ad0040573b0c00f88488bc31958acd07a55ee2e Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 8 Oct 2015 01:20:39 +0200 Subject: [PATCH 5/5] bpf: split state from prandom_u32() and consolidate {c, e}BPF prngs While recently arguing on a seccomp discussion that raw prandom_u32() access shouldn't be exposed to unpriviledged user space, I forgot the fact that SKF_AD_RANDOM extension actually already does it for some time in cBPF via commit 4cd3675ebf74 ("filter: added BPF random opcode"). Since prandom_u32() is being used in a lot of critical networking code, lets be more conservative and split their states. Furthermore, consolidate eBPF and cBPF prandom handlers to use the new internal PRNG. For eBPF, bpf_get_prandom_u32() was only accessible for priviledged users, but should that change one day, we also don't want to leak raw sequences through things like eBPF maps. One thought was also to have own per bpf_prog states, but due to ABI reasons this is not easily possible, i.e. the program code currently cannot access bpf_prog itself, and copying the rnd_state to/from the stack scratch space whenever a program uses the prng seems not really worth the trouble and seems too hacky. If needed, taus113 could in such cases be implemented within eBPF using a map entry to keep the state space, or get_random_bytes() could become a second helper in cases where performance would not be critical. Both sides can trigger a one-time late init via prandom_init_once() on the shared state. Performance-wise, there should even be a tiny gain as bpf_user_rnd_u32() saves one function call. The PRNG needs to live inside the BPF core since kernels could have a NET-less config as well. Signed-off-by: Daniel Borkmann Acked-by: Hannes Frederic Sowa Acked-by: Alexei Starovoitov Cc: Chema Gonzalez Signed-off-by: David S. Miller --- include/linux/bpf.h | 4 ++++ kernel/bpf/core.c | 26 ++++++++++++++++++++++++++ kernel/bpf/helpers.c | 7 +------ kernel/bpf/syscall.c | 2 ++ net/core/filter.c | 9 ++------- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c915a6b54570c..3697ad5638996 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -200,4 +200,8 @@ extern const struct bpf_func_proto bpf_get_current_comm_proto; extern const struct bpf_func_proto bpf_skb_vlan_push_proto; extern const struct bpf_func_proto bpf_skb_vlan_pop_proto; +/* Shared helpers among cBPF and eBPF. */ +void bpf_user_rnd_init_once(void); +u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); + #endif /* _LINUX_BPF_H */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index c8855c2a7a480..80864712d2c40 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -731,6 +731,32 @@ void bpf_prog_free(struct bpf_prog *fp) } EXPORT_SYMBOL_GPL(bpf_prog_free); +/* RNG for unpriviledged user space with separated state from prandom_u32(). */ +static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state); + +void bpf_user_rnd_init_once(void) +{ + prandom_init_once(&bpf_user_rnd_state); +} + +u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) +{ + /* Should someone ever have the rather unwise idea to use some + * of the registers passed into this function, then note that + * this function is called from native eBPF and classic-to-eBPF + * transformations. Register assignments from both sides are + * different, f.e. classic always sets fn(ctx, A, X) here. + */ + struct rnd_state *state; + u32 res; + + state = &get_cpu_var(bpf_user_rnd_state); + res = prandom_u32_state(state); + put_cpu_var(state); + + return res; +} + /* Weak definitions of helper functions in case we don't have bpf syscall. */ const struct bpf_func_proto bpf_map_lookup_elem_proto __weak; const struct bpf_func_proto bpf_map_update_elem_proto __weak; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 1447ec09421eb..4504ca66118da 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -93,13 +93,8 @@ const struct bpf_func_proto bpf_map_delete_elem_proto = { .arg2_type = ARG_PTR_TO_MAP_KEY, }; -static u64 bpf_get_prandom_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -{ - return prandom_u32(); -} - const struct bpf_func_proto bpf_get_prandom_u32_proto = { - .func = bpf_get_prandom_u32, + .func = bpf_user_rnd_u32, .gpl_only = false, .ret_type = RET_INTEGER, }; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 5f35f420c12fa..c868cafbc00c6 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -404,6 +404,8 @@ static void fixup_bpf_calls(struct bpf_prog *prog) if (insn->imm == BPF_FUNC_get_route_realm) prog->dst_needed = 1; + if (insn->imm == BPF_FUNC_get_prandom_u32) + bpf_user_rnd_init_once(); if (insn->imm == BPF_FUNC_tail_call) { /* mark bpf_tail_call as different opcode * to avoid conditional branch in diff --git a/net/core/filter.c b/net/core/filter.c index 8f4603c712cd0..342e6c8fc415b 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -149,12 +149,6 @@ static u64 __get_raw_cpu_id(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) return raw_smp_processor_id(); } -/* note that this only generates 32-bit random numbers */ -static u64 __get_random_u32(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) -{ - return prandom_u32(); -} - static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg, struct bpf_insn *insn_buf) { @@ -313,7 +307,8 @@ static bool convert_bpf_extensions(struct sock_filter *fp, *insn = BPF_EMIT_CALL(__get_raw_cpu_id); break; case SKF_AD_OFF + SKF_AD_RANDOM: - *insn = BPF_EMIT_CALL(__get_random_u32); + *insn = BPF_EMIT_CALL(bpf_user_rnd_u32); + bpf_user_rnd_init_once(); break; } break;