Skip to content

Commit

Permalink
net: keep sk->sk_forward_alloc as small as possible
Browse files Browse the repository at this point in the history
Currently, tcp_memory_allocated can hit tcp_mem[] limits quite fast.

Each TCP socket can forward allocate up to 2 MB of memory, even after
flow became less active.

10,000 sockets can have reserved 20 GB of memory,
and we have no shrinker in place to reclaim that.

Instead of trying to reclaim the extra allocations in some places,
just keep sk->sk_forward_alloc values as small as possible.

This should not impact performance too much now we have per-cpu
reserves: Changes to tcp_memory_allocated should not be too frequent.

For sockets not using SO_RESERVE_MEM:
 - idle sockets (no packets in tx/rx queues) have zero forward alloc.
 - non idle sockets have a forward alloc smaller than one page.

Note:

 - Removal of SK_RECLAIM_CHUNK and SK_RECLAIM_THRESHOLD
   is left to MPTCP maintainers as a follow up.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Shakeel Butt <shakeelb@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
  • Loading branch information
Eric Dumazet authored and Jakub Kicinski committed Jun 10, 2022
1 parent 7c80b03 commit 4890b68
Show file tree
Hide file tree
Showing 11 changed files with 7 additions and 72 deletions.
29 changes: 2 additions & 27 deletions include/net/sock.h
Original file line number Diff line number Diff line change
Expand Up @@ -1627,49 +1627,24 @@ static inline void sk_mem_reclaim_final(struct sock *sk)
sk_mem_reclaim(sk);
}

static inline void sk_mem_reclaim_partial(struct sock *sk)
{
int reclaimable;

if (!sk_has_account(sk))
return;

reclaimable = sk->sk_forward_alloc - sk_unused_reserved_mem(sk);

if (reclaimable > (int)PAGE_SIZE)
__sk_mem_reclaim(sk, reclaimable - 1);
}

static inline void sk_mem_charge(struct sock *sk, int size)
{
if (!sk_has_account(sk))
return;
sk->sk_forward_alloc -= size;
}

/* the following macros control memory reclaiming in sk_mem_uncharge()
/* the following macros control memory reclaiming in mptcp_rmem_uncharge()
*/
#define SK_RECLAIM_THRESHOLD (1 << 21)
#define SK_RECLAIM_CHUNK (1 << 20)

static inline void sk_mem_uncharge(struct sock *sk, int size)
{
int reclaimable;

if (!sk_has_account(sk))
return;
sk->sk_forward_alloc += size;
reclaimable = sk->sk_forward_alloc - sk_unused_reserved_mem(sk);

/* Avoid a possible overflow.
* TCP send queues can make this happen, if sk_mem_reclaim()
* is not called and more than 2 GBytes are released at once.
*
* If we reach 2 MBytes, reclaim 1 MBytes right now, there is
* no need to hold that much forward allocation anyway.
*/
if (unlikely(reclaimable >= SK_RECLAIM_THRESHOLD))
__sk_mem_reclaim(sk, SK_RECLAIM_CHUNK);
sk_mem_reclaim(sk);
}

/*
Expand Down
3 changes: 0 additions & 3 deletions net/core/datagram.c
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,6 @@ EXPORT_SYMBOL(skb_recv_datagram);
void skb_free_datagram(struct sock *sk, struct sk_buff *skb)
{
consume_skb(skb);
sk_mem_reclaim_partial(sk);
}
EXPORT_SYMBOL(skb_free_datagram);

Expand All @@ -336,7 +335,6 @@ void __skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb, int len)
slow = lock_sock_fast(sk);
sk_peek_offset_bwd(sk, len);
skb_orphan(skb);
sk_mem_reclaim_partial(sk);
unlock_sock_fast(sk, slow);

/* skb is now orphaned, can be freed outside of locked section */
Expand Down Expand Up @@ -396,7 +394,6 @@ int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags)
NULL);

kfree_skb(skb);
sk_mem_reclaim_partial(sk);
return err;
}
EXPORT_SYMBOL(skb_kill_datagram);
Expand Down
7 changes: 0 additions & 7 deletions net/ipv4/tcp.c
Original file line number Diff line number Diff line change
Expand Up @@ -858,9 +858,6 @@ struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
{
struct sk_buff *skb;

if (unlikely(tcp_under_memory_pressure(sk)))
sk_mem_reclaim_partial(sk);

skb = alloc_skb_fclone(size + MAX_TCP_HEADER, gfp);
if (likely(skb)) {
bool mem_scheduled;
Expand Down Expand Up @@ -2764,8 +2761,6 @@ void __tcp_close(struct sock *sk, long timeout)
__kfree_skb(skb);
}

sk_mem_reclaim(sk);

/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
if (sk->sk_state == TCP_CLOSE)
goto adjudge_to_death;
Expand Down Expand Up @@ -2873,7 +2868,6 @@ void __tcp_close(struct sock *sk, long timeout)
}
}
if (sk->sk_state != TCP_CLOSE) {
sk_mem_reclaim(sk);
if (tcp_check_oom(sk, 0)) {
tcp_set_state(sk, TCP_CLOSE);
tcp_send_active_reset(sk, GFP_ATOMIC);
Expand Down Expand Up @@ -2951,7 +2945,6 @@ void tcp_write_queue_purge(struct sock *sk)
}
tcp_rtx_queue_purge(sk);
INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue);
sk_mem_reclaim(sk);
tcp_clear_all_retrans_hints(tcp_sk(sk));
tcp_sk(sk)->packets_out = 0;
inet_csk(sk)->icsk_backoff = 0;
Expand Down
4 changes: 0 additions & 4 deletions net/ipv4/tcp_input.c
Original file line number Diff line number Diff line change
Expand Up @@ -805,7 +805,6 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
* restart window, so that we send ACKs quickly.
*/
tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
sk_mem_reclaim(sk);
}
}
icsk->icsk_ack.lrcvtime = now;
Expand Down Expand Up @@ -4390,7 +4389,6 @@ void tcp_fin(struct sock *sk)
skb_rbtree_purge(&tp->out_of_order_queue);
if (tcp_is_sack(tp))
tcp_sack_reset(&tp->rx_opt);
sk_mem_reclaim(sk);

if (!sock_flag(sk, SOCK_DEAD)) {
sk->sk_state_change(sk);
Expand Down Expand Up @@ -5336,7 +5334,6 @@ static bool tcp_prune_ofo_queue(struct sock *sk)
tcp_drop_reason(sk, rb_to_skb(node),
SKB_DROP_REASON_TCP_OFO_QUEUE_PRUNE);
if (!prev || goal <= 0) {
sk_mem_reclaim(sk);
if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf &&
!tcp_under_memory_pressure(sk))
break;
Expand Down Expand Up @@ -5383,7 +5380,6 @@ static int tcp_prune_queue(struct sock *sk)
skb_peek(&sk->sk_receive_queue),
NULL,
tp->copied_seq, tp->rcv_nxt);
sk_mem_reclaim(sk);

if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
return 0;
Expand Down
19 changes: 4 additions & 15 deletions net/ipv4/tcp_timer.c
Original file line number Diff line number Diff line change
Expand Up @@ -290,15 +290,13 @@ void tcp_delack_timer_handler(struct sock *sk)
{
struct inet_connection_sock *icsk = inet_csk(sk);

sk_mem_reclaim_partial(sk);

if (((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) ||
!(icsk->icsk_ack.pending & ICSK_ACK_TIMER))
goto out;
return;

if (time_after(icsk->icsk_ack.timeout, jiffies)) {
sk_reset_timer(sk, &icsk->icsk_delack_timer, icsk->icsk_ack.timeout);
goto out;
return;
}
icsk->icsk_ack.pending &= ~ICSK_ACK_TIMER;

Expand All @@ -317,10 +315,6 @@ void tcp_delack_timer_handler(struct sock *sk)
tcp_send_ack(sk);
__NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKS);
}

out:
if (tcp_under_memory_pressure(sk))
sk_mem_reclaim(sk);
}


Expand Down Expand Up @@ -600,11 +594,11 @@ void tcp_write_timer_handler(struct sock *sk)

if (((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) ||
!icsk->icsk_pending)
goto out;
return;

if (time_after(icsk->icsk_timeout, jiffies)) {
sk_reset_timer(sk, &icsk->icsk_retransmit_timer, icsk->icsk_timeout);
goto out;
return;
}

tcp_mstamp_refresh(tcp_sk(sk));
Expand All @@ -626,9 +620,6 @@ void tcp_write_timer_handler(struct sock *sk)
tcp_probe_timer(sk);
break;
}

out:
sk_mem_reclaim(sk);
}

static void tcp_write_timer(struct timer_list *t)
Expand Down Expand Up @@ -743,8 +734,6 @@ static void tcp_keepalive_timer (struct timer_list *t)
elapsed = keepalive_time_when(tp) - elapsed;
}

sk_mem_reclaim(sk);

resched:
inet_csk_reset_keepalive_timer (sk, elapsed);
goto out;
Expand Down
2 changes: 0 additions & 2 deletions net/iucv/af_iucv.c
Original file line number Diff line number Diff line change
Expand Up @@ -278,8 +278,6 @@ static void iucv_sock_destruct(struct sock *sk)
skb_queue_purge(&sk->sk_receive_queue);
skb_queue_purge(&sk->sk_error_queue);

sk_mem_reclaim(sk);

if (!sock_flag(sk, SOCK_DEAD)) {
pr_err("Attempt to release alive iucv socket %p\n", sk);
return;
Expand Down
2 changes: 1 addition & 1 deletion net/mptcp/protocol.c
Original file line number Diff line number Diff line change
Expand Up @@ -975,7 +975,7 @@ static void __mptcp_mem_reclaim_partial(struct sock *sk)
if (reclaimable > (int)PAGE_SIZE)
__mptcp_rmem_reclaim(sk, reclaimable - 1);

sk_mem_reclaim_partial(sk);
sk_mem_reclaim(sk);
}

static void mptcp_mem_reclaim_partial(struct sock *sk)
Expand Down
2 changes: 0 additions & 2 deletions net/sctp/sm_statefuns.c
Original file line number Diff line number Diff line change
Expand Up @@ -6590,8 +6590,6 @@ static int sctp_eat_data(const struct sctp_association *asoc,
pr_debug("%s: under pressure, reneging for tsn:%u\n",
__func__, tsn);
deliver = SCTP_CMD_RENEGE;
} else {
sk_mem_reclaim(sk);
}
}

Expand Down
5 changes: 0 additions & 5 deletions net/sctp/socket.c
Original file line number Diff line number Diff line change
Expand Up @@ -1824,9 +1824,6 @@ static int sctp_sendmsg_to_asoc(struct sctp_association *asoc,
if (sctp_wspace(asoc) < (int)msg_len)
sctp_prsctp_prune(asoc, sinfo, msg_len - sctp_wspace(asoc));

if (sk_under_memory_pressure(sk))
sk_mem_reclaim(sk);

if (sctp_wspace(asoc) <= 0 || !sk_wmem_schedule(sk, msg_len)) {
timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
err = sctp_wait_for_sndbuf(asoc, &timeo, msg_len);
Expand Down Expand Up @@ -9195,8 +9192,6 @@ static int sctp_wait_for_sndbuf(struct sctp_association *asoc, long *timeo_p,
goto do_error;
if (signal_pending(current))
goto do_interrupted;
if (sk_under_memory_pressure(sk))
sk_mem_reclaim(sk);
if ((int)msg_len <= sctp_wspace(asoc) &&
sk_wmem_schedule(sk, msg_len))
break;
Expand Down
2 changes: 0 additions & 2 deletions net/sctp/stream_interleave.c
Original file line number Diff line number Diff line change
Expand Up @@ -979,8 +979,6 @@ static void sctp_renege_events(struct sctp_ulpq *ulpq, struct sctp_chunk *chunk,

if (freed >= needed && sctp_ulpevent_idata(ulpq, chunk, gfp) <= 0)
sctp_intl_start_pd(ulpq, gfp);

sk_mem_reclaim(asoc->base.sk);
}

static void sctp_intl_stream_abort_pd(struct sctp_ulpq *ulpq, __u16 sid,
Expand Down
4 changes: 0 additions & 4 deletions net/sctp/ulpqueue.c
Original file line number Diff line number Diff line change
Expand Up @@ -1100,12 +1100,8 @@ void sctp_ulpq_renege(struct sctp_ulpq *ulpq, struct sctp_chunk *chunk,
else if (retval == 1)
sctp_ulpq_reasm_drain(ulpq);
}

sk_mem_reclaim(asoc->base.sk);
}



/* Notify the application if an association is aborted and in
* partial delivery mode. Send up any pending received messages.
*/
Expand Down

0 comments on commit 4890b68

Please sign in to comment.