Skip to content

Commit

Permalink
Merge branch 'tcp-tx-headless'
Browse files Browse the repository at this point in the history 
Eric Dumazet says:

====================
tcp: tx path fully headless

This series completes transition of TCP stack tx path
to headless packets : All payload now reside in page frags,
never in skb->head.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
David S. Miller committed Jun 12, 2023
2 parents f2f069d + 5882eff commit 73f49f8
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 48 deletions.
3 changes: 2 additions & 1 deletion include/net/tcp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,
int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
size_t size, int flags);
int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
int tcp_wmem_schedule(struct sock *sk, int copy);
void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
int size_goal);
void tcp_release_cb(struct sock *sk);
Expand All @@ -349,7 +350,7 @@ void tcp_twsk_purge(struct list_head *net_exit_list, int family);
ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
struct pipe_inode_info *pipe, size_t len,
unsigned int flags);
struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
bool force_schedule);

void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks);
Expand Down
8 changes: 4 additions & 4 deletions net/ipv4/tcp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -858,12 +858,12 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
}
EXPORT_SYMBOL(tcp_splice_read);

struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
bool force_schedule)
{
struct sk_buff *skb;

skb = alloc_skb_fclone(size + MAX_TCP_HEADER, gfp);
skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp);
if (likely(skb)) {
bool mem_scheduled;

Expand Down Expand Up @@ -957,7 +957,7 @@ static int tcp_downgrade_zcopy_pure(struct sock *sk, struct sk_buff *skb)
}


static int tcp_wmem_schedule(struct sock *sk, int copy)
int tcp_wmem_schedule(struct sock *sk, int copy)
{
int left;

Expand Down Expand Up @@ -1178,7 +1178,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
goto restart;
}
first_skb = tcp_rtx_and_write_queues_empty(sk);
skb = tcp_stream_alloc_skb(sk, 0, sk->sk_allocation,
skb = tcp_stream_alloc_skb(sk, sk->sk_allocation,
first_skb);
if (!skb)
goto wait_for_space;
Expand Down
77 changes: 34 additions & 43 deletions net/ipv4/tcp_output.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1530,17 +1530,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
{
struct tcp_sock *tp = tcp_sk(sk);
struct sk_buff *buff;
int nsize, old_factor;
int old_factor;
long limit;
int nlen;
u8 flags;

if (WARN_ON(len > skb->len))
return -EINVAL;

nsize = skb_headlen(skb) - len;
if (nsize < 0)
nsize = 0;
DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb));

/* tcp_sendmsg() can overshoot sk_wmem_queued by one full size skb.
* We need some allowance to not penalize applications setting small
Expand All @@ -1560,15 +1558,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
return -ENOMEM;

/* Get a new skb... force flag on. */
buff = tcp_stream_alloc_skb(sk, nsize, gfp, true);
buff = tcp_stream_alloc_skb(sk, gfp, true);
if (!buff)
return -ENOMEM; /* We'll just try again later. */
skb_copy_decrypted(buff, skb);
mptcp_skb_ext_copy(buff, skb);

sk_wmem_queued_add(sk, buff->truesize);
sk_mem_charge(sk, buff->truesize);
nlen = skb->len - len - nsize;
nlen = skb->len - len;
buff->truesize += nlen;
skb->truesize -= nlen;

Expand Down Expand Up @@ -1626,13 +1624,7 @@ static int __pskb_trim_head(struct sk_buff *skb, int len)
struct skb_shared_info *shinfo;
int i, k, eat;

eat = min_t(int, len, skb_headlen(skb));
if (eat) {
__skb_pull(skb, eat);
len -= eat;
if (!len)
return 0;
}
DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb));
eat = len;
k = 0;
shinfo = skb_shinfo(skb);
Expand Down Expand Up @@ -1671,12 +1663,10 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)

TCP_SKB_CB(skb)->seq += len;

if (delta_truesize) {
skb->truesize -= delta_truesize;
sk_wmem_queued_add(sk, -delta_truesize);
if (!skb_zcopy_pure(skb))
sk_mem_uncharge(sk, delta_truesize);
}
skb->truesize -= delta_truesize;
sk_wmem_queued_add(sk, -delta_truesize);
if (!skb_zcopy_pure(skb))
sk_mem_uncharge(sk, delta_truesize);

/* Any change of skb->len requires recalculation of tso factor. */
if (tcp_skb_pcount(skb) > 1)
Expand Down Expand Up @@ -2126,11 +2116,9 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
u8 flags;

/* All of a TSO frame must be composed of paged data. */
if (skb->len != skb->data_len)
return tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
skb, len, mss_now, gfp);
DEBUG_NET_WARN_ON_ONCE(skb->len != skb->data_len);

buff = tcp_stream_alloc_skb(sk, 0, gfp, true);
buff = tcp_stream_alloc_skb(sk, gfp, true);
if (unlikely(!buff))
return -ENOMEM;
skb_copy_decrypted(buff, skb);
Expand Down Expand Up @@ -2446,7 +2434,7 @@ static int tcp_mtu_probe(struct sock *sk)
return -1;

/* We're allowed to probe. Build it now. */
nskb = tcp_stream_alloc_skb(sk, 0, GFP_ATOMIC, false);
nskb = tcp_stream_alloc_skb(sk, GFP_ATOMIC, false);
if (!nskb)
return -1;

Expand Down Expand Up @@ -2487,12 +2475,8 @@ static int tcp_mtu_probe(struct sock *sk)
} else {
TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
~(TCPHDR_FIN|TCPHDR_PSH);
if (!skb_shinfo(skb)->nr_frags) {
skb_pull(skb, copy);
} else {
__pskb_trim_head(skb, copy);
tcp_set_skb_tso_segs(skb, mss_now);
}
__pskb_trim_head(skb, copy);
tcp_set_skb_tso_segs(skb, mss_now);
TCP_SKB_CB(skb)->seq += copy;
}

Expand Down Expand Up @@ -3802,8 +3786,9 @@ static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
struct inet_connection_sock *icsk = inet_csk(sk);
struct tcp_sock *tp = tcp_sk(sk);
struct tcp_fastopen_request *fo = tp->fastopen_req;
int space, err = 0;
struct page_frag *pfrag = sk_page_frag(sk);
struct sk_buff *syn_data;
int space, err = 0;

tp->rx_opt.mss_clamp = tp->advmss; /* If MSS is not cached */
if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie))
Expand All @@ -3822,25 +3807,31 @@ static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)

space = min_t(size_t, space, fo->size);

/* limit to order-0 allocations */
space = min_t(size_t, space, SKB_MAX_HEAD(MAX_TCP_HEADER));

syn_data = tcp_stream_alloc_skb(sk, space, sk->sk_allocation, false);
if (space &&
!skb_page_frag_refill(min_t(size_t, space, PAGE_SIZE),
pfrag, sk->sk_allocation))
goto fallback;
syn_data = tcp_stream_alloc_skb(sk, sk->sk_allocation, false);
if (!syn_data)
goto fallback;
memcpy(syn_data->cb, syn->cb, sizeof(syn->cb));
if (space) {
int copied = copy_from_iter(skb_put(syn_data, space), space,
&fo->data->msg_iter);
if (unlikely(!copied)) {
space = min_t(size_t, space, pfrag->size - pfrag->offset);
space = tcp_wmem_schedule(sk, space);
}
if (space) {
space = copy_page_from_iter(pfrag->page, pfrag->offset,
space, &fo->data->msg_iter);
if (unlikely(!space)) {
tcp_skb_tsorted_anchor_cleanup(syn_data);
kfree_skb(syn_data);
goto fallback;
}
if (copied != space) {
skb_trim(syn_data, copied);
space = copied;
}
skb_fill_page_desc(syn_data, 0, pfrag->page,
pfrag->offset, space);
page_ref_inc(pfrag->page);
pfrag->offset += space;
skb_len_add(syn_data, space);
skb_zcopy_set(syn_data, fo->uarg, NULL);
}
/* No more data pending in inet_wait_for_connect() */
Expand Down Expand Up @@ -3905,7 +3896,7 @@ int tcp_connect(struct sock *sk)
return 0;
}

buff = tcp_stream_alloc_skb(sk, 0, sk->sk_allocation, true);
buff = tcp_stream_alloc_skb(sk, sk->sk_allocation, true);
if (unlikely(!buff))
return -ENOBUFS;

Expand Down

0 comments on commit 73f49f8

Please sign in to comment.