Skip to content

Commit

Permalink
Merge branch 'tls-rx-decrypt-from-the-tcp-queue'
Browse files Browse the repository at this point in the history
Jakub Kicinski says:

====================
tls: rx: decrypt from the TCP queue

This is the final part of my TLS Rx rework. It switches from
strparser to decrypting data from skbs queued in TCP. We don't
need the full strparser for TLS, its needs are very basic.
This set gives us a small but measurable (6%) performance
improvement (continuous stream).
====================

Link: https://lore.kernel.org/r/20220722235033.2594446-1-kuba@kernel.org
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
  • Loading branch information
Jakub Kicinski committed Jul 26, 2022
2 parents 4fd310c + 84c61fe commit 48c022d
Show file tree
Hide file tree
Showing 8 changed files with 723 additions and 124 deletions.
2 changes: 2 additions & 0 deletions include/net/tcp.h
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,8 @@ void tcp_get_info(struct sock *, struct tcp_info *);
int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
sk_read_actor_t recv_actor);
int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off);
void tcp_read_done(struct sock *sk, size_t len);

void tcp_initialize_rcv_mss(struct sock *sk);

Expand Down
19 changes: 17 additions & 2 deletions include/net/tls.h
Original file line number Diff line number Diff line change
Expand Up @@ -108,18 +108,33 @@ struct tls_sw_context_tx {
unsigned long tx_bitmask;
};

struct tls_strparser {
struct sock *sk;

u32 mark : 8;
u32 stopped : 1;
u32 copy_mode : 1;
u32 msg_ready : 1;

struct strp_msg stm;

struct sk_buff *anchor;
struct work_struct work;
};

struct tls_sw_context_rx {
struct crypto_aead *aead_recv;
struct crypto_wait async_wait;
struct strparser strp;
struct sk_buff_head rx_list; /* list of decrypted 'data' records */
void (*saved_data_ready)(struct sock *sk);

struct sk_buff *recv_pkt;
u8 reader_present;
u8 async_capable:1;
u8 zc_capable:1;
u8 reader_contended:1;

struct tls_strparser strp;

atomic_t decrypt_pending;
/* protect crypto_wait with decrypt_pending*/
spinlock_t decrypt_compl_lock;
Expand Down
42 changes: 41 additions & 1 deletion net/ipv4/tcp.c
Original file line number Diff line number Diff line change
Expand Up @@ -1635,7 +1635,7 @@ static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb)
__kfree_skb(skb);
}

static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
{
struct sk_buff *skb;
u32 offset;
Expand All @@ -1658,6 +1658,7 @@ static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
}
return NULL;
}
EXPORT_SYMBOL(tcp_recv_skb);

/*
* This routine provides an alternative to tcp_recvmsg() for routines
Expand Down Expand Up @@ -1788,6 +1789,45 @@ int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
}
EXPORT_SYMBOL(tcp_read_skb);

void tcp_read_done(struct sock *sk, size_t len)
{
struct tcp_sock *tp = tcp_sk(sk);
u32 seq = tp->copied_seq;
struct sk_buff *skb;
size_t left;
u32 offset;

if (sk->sk_state == TCP_LISTEN)
return;

left = len;
while (left && (skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
int used;

used = min_t(size_t, skb->len - offset, left);
seq += used;
left -= used;

if (skb->len > offset + used)
break;

if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
tcp_eat_recv_skb(sk, skb);
++seq;
break;
}
tcp_eat_recv_skb(sk, skb);
}
WRITE_ONCE(tp->copied_seq, seq);

tcp_rcv_space_adjust(sk);

/* Clean up data we have read: This will do ACK frames. */
if (left != len)
tcp_cleanup_rbuf(sk, len - left);
}
EXPORT_SYMBOL(tcp_read_done);

int tcp_peek_len(struct socket *sock)
{
return tcp_inq(sock->sk);
Expand Down
29 changes: 26 additions & 3 deletions net/tls/tls.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
/*
* Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
* Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
* Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
*
Expand Down Expand Up @@ -127,8 +128,24 @@ int tls_sw_fallback_init(struct sock *sk,
struct tls_offload_context_tx *offload_ctx,
struct tls_crypto_info *crypto_info);

int tls_strp_msg_hold(struct sock *sk, struct sk_buff *skb,
struct sk_buff_head *dst);
int tls_strp_dev_init(void);
void tls_strp_dev_exit(void);

void tls_strp_done(struct tls_strparser *strp);
void tls_strp_stop(struct tls_strparser *strp);
int tls_strp_init(struct tls_strparser *strp, struct sock *sk);
void tls_strp_data_ready(struct tls_strparser *strp);

void tls_strp_check_rcv(struct tls_strparser *strp);
void tls_strp_msg_done(struct tls_strparser *strp);

int tls_rx_msg_size(struct tls_strparser *strp, struct sk_buff *skb);
void tls_rx_msg_ready(struct tls_strparser *strp);

void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh);
int tls_strp_msg_cow(struct tls_sw_context_rx *ctx);
struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx);
int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst);

static inline struct tls_msg *tls_msg(struct sk_buff *skb)
{
Expand All @@ -139,7 +156,13 @@ static inline struct tls_msg *tls_msg(struct sk_buff *skb)

static inline struct sk_buff *tls_strp_msg(struct tls_sw_context_rx *ctx)
{
return ctx->recv_pkt;
DEBUG_NET_WARN_ON_ONCE(!ctx->strp.msg_ready || !ctx->strp.anchor->len);
return ctx->strp.anchor;
}

static inline bool tls_strp_msg_ready(struct tls_sw_context_rx *ctx)
{
return ctx->strp.msg_ready;
}

#ifdef CONFIG_TLS_DEVICE
Expand Down
19 changes: 9 additions & 10 deletions net/tls/tls_device.c
Original file line number Diff line number Diff line change
Expand Up @@ -894,27 +894,26 @@ static void tls_device_core_ctrl_rx_resync(struct tls_context *tls_ctx,
static int
tls_device_reencrypt(struct sock *sk, struct tls_sw_context_rx *sw_ctx)
{
int err = 0, offset, copy, nsg, data_len, pos;
struct sk_buff *skb, *skb_iter, *unused;
int err, offset, copy, data_len, pos;
struct sk_buff *skb, *skb_iter;
struct scatterlist sg[1];
struct strp_msg *rxm;
char *orig_buf, *buf;

skb = tls_strp_msg(sw_ctx);
rxm = strp_msg(skb);
offset = rxm->offset;

rxm = strp_msg(tls_strp_msg(sw_ctx));
orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE +
TLS_CIPHER_AES_GCM_128_IV_SIZE, sk->sk_allocation);
if (!orig_buf)
return -ENOMEM;
buf = orig_buf;

nsg = skb_cow_data(skb, 0, &unused);
if (unlikely(nsg < 0)) {
err = nsg;
err = tls_strp_msg_cow(sw_ctx);
if (unlikely(err))
goto free_buf;
}

skb = tls_strp_msg(sw_ctx);
rxm = strp_msg(skb);
offset = rxm->offset;

sg_init_table(sg, 1);
sg_set_buf(&sg[0], buf,
Expand Down
20 changes: 16 additions & 4 deletions net/tls/tls_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,10 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
if (tx) {
ctx->sk_write_space = sk->sk_write_space;
sk->sk_write_space = tls_write_space;
} else {
struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(ctx);

tls_strp_check_rcv(&rx_ctx->strp);
}
return 0;

Expand Down Expand Up @@ -1141,20 +1145,28 @@ static int __init tls_register(void)
if (err)
return err;

err = tls_strp_dev_init();
if (err)
goto err_pernet;

err = tls_device_init();
if (err) {
unregister_pernet_subsys(&tls_proc_ops);
return err;
}
if (err)
goto err_strp;

tcp_register_ulp(&tcp_tls_ulp_ops);

return 0;
err_strp:
tls_strp_dev_exit();
err_pernet:
unregister_pernet_subsys(&tls_proc_ops);
return err;
}

static void __exit tls_unregister(void)
{
tcp_unregister_ulp(&tcp_tls_ulp_ops);
tls_strp_dev_exit();
tls_device_cleanup();
unregister_pernet_subsys(&tls_proc_ops);
}
Expand Down
Loading

0 comments on commit 48c022d

Please sign in to comment.