Skip to content

Commit

Permalink
tls: rx: do not use the standard strparser
Browse files Browse the repository at this point in the history
TLS is a relatively poor fit for strparser. We pause the input
every time a message is received, wait for a read which will
decrypt the message, start the parser, repeat. strparser is
built to delineate the messages, wrap them in individual skbs
and let them float off into the stack or a different socket.
TLS wants the data pages and nothing else. There's no need
for TLS to keep cloning (and occasionally skb_unclone()'ing)
the TCP rx queue.

This patch uses a pre-allocated skb and attaches the skbs
from the TCP rx queue to it as frags. TLS is careful never
to modify the input skb without CoW'ing / detaching it first.

Since we call TCP rx queue cleanup directly we also get back
the benefit of skb deferred free.

Overall this results in a 6% gain in my benchmarks.

Acked-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
  • Loading branch information
Jakub Kicinski committed Jul 26, 2022
1 parent 8b3c59a commit 84c61fe
Show file tree
Hide file tree
Showing 5 changed files with 558 additions and 69 deletions.
19 changes: 17 additions & 2 deletions include/net/tls.h
Original file line number Diff line number Diff line change
Expand Up @@ -108,18 +108,33 @@ struct tls_sw_context_tx {
unsigned long tx_bitmask;
};

struct tls_strparser {
struct sock *sk;

u32 mark : 8;
u32 stopped : 1;
u32 copy_mode : 1;
u32 msg_ready : 1;

struct strp_msg stm;

struct sk_buff *anchor;
struct work_struct work;
};

struct tls_sw_context_rx {
struct crypto_aead *aead_recv;
struct crypto_wait async_wait;
struct strparser strp;
struct sk_buff_head rx_list; /* list of decrypted 'data' records */
void (*saved_data_ready)(struct sock *sk);

struct sk_buff *recv_pkt;
u8 reader_present;
u8 async_capable:1;
u8 zc_capable:1;
u8 reader_contended:1;

struct tls_strparser strp;

atomic_t decrypt_pending;
/* protect crypto_wait with decrypt_pending*/
spinlock_t decrypt_compl_lock;
Expand Down
24 changes: 20 additions & 4 deletions net/tls/tls.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
/*
* Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
* Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
* Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
*
Expand Down Expand Up @@ -127,10 +128,24 @@ int tls_sw_fallback_init(struct sock *sk,
struct tls_offload_context_tx *offload_ctx,
struct tls_crypto_info *crypto_info);

int tls_strp_dev_init(void);
void tls_strp_dev_exit(void);

void tls_strp_done(struct tls_strparser *strp);
void tls_strp_stop(struct tls_strparser *strp);
int tls_strp_init(struct tls_strparser *strp, struct sock *sk);
void tls_strp_data_ready(struct tls_strparser *strp);

void tls_strp_check_rcv(struct tls_strparser *strp);
void tls_strp_msg_done(struct tls_strparser *strp);

int tls_rx_msg_size(struct tls_strparser *strp, struct sk_buff *skb);
void tls_rx_msg_ready(struct tls_strparser *strp);

void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh);
int tls_strp_msg_cow(struct tls_sw_context_rx *ctx);
struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx);
int tls_strp_msg_hold(struct sock *sk, struct sk_buff *skb,
struct sk_buff_head *dst);
int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst);

static inline struct tls_msg *tls_msg(struct sk_buff *skb)
{
Expand All @@ -141,12 +156,13 @@ static inline struct tls_msg *tls_msg(struct sk_buff *skb)

static inline struct sk_buff *tls_strp_msg(struct tls_sw_context_rx *ctx)
{
return ctx->recv_pkt;
DEBUG_NET_WARN_ON_ONCE(!ctx->strp.msg_ready || !ctx->strp.anchor->len);
return ctx->strp.anchor;
}

static inline bool tls_strp_msg_ready(struct tls_sw_context_rx *ctx)
{
return ctx->recv_pkt;
return ctx->strp.msg_ready;
}

#ifdef CONFIG_TLS_DEVICE
Expand Down
20 changes: 16 additions & 4 deletions net/tls/tls_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,10 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
if (tx) {
ctx->sk_write_space = sk->sk_write_space;
sk->sk_write_space = tls_write_space;
} else {
struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(ctx);

tls_strp_check_rcv(&rx_ctx->strp);
}
return 0;

Expand Down Expand Up @@ -1141,20 +1145,28 @@ static int __init tls_register(void)
if (err)
return err;

err = tls_strp_dev_init();
if (err)
goto err_pernet;

err = tls_device_init();
if (err) {
unregister_pernet_subsys(&tls_proc_ops);
return err;
}
if (err)
goto err_strp;

tcp_register_ulp(&tcp_tls_ulp_ops);

return 0;
err_strp:
tls_strp_dev_exit();
err_pernet:
unregister_pernet_subsys(&tls_proc_ops);
return err;
}

static void __exit tls_unregister(void)
{
tcp_unregister_ulp(&tcp_tls_ulp_ops);
tls_strp_dev_exit();
tls_device_cleanup();
unregister_pernet_subsys(&tls_proc_ops);
}
Expand Down
Loading

0 comments on commit 84c61fe

Please sign in to comment.