Skip to content

Commit

Permalink
net: Use a percpu_counter for sockets_allocated
Browse files Browse the repository at this point in the history
Instead of using one atomic_t per protocol, use a percpu_counter
for "sockets_allocated", to reduce cache line contention on
heavy duty network servers. 

Note : We revert commit (248969a
net: af_unix can make unix_nr_socks visbile in /proc),
since it is not anymore used after sock_prot_inuse_add() addition

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Eric Dumazet authored and David S. Miller committed Nov 26, 2008
1 parent c1b5687 commit 1748376
Showing 11 changed files with 29 additions and 16 deletions.
1 change: 1 addition & 0 deletions include/net/sctp/sctp.h
Original file line number Diff line number Diff line change
@@ -138,6 +138,7 @@ void sctp_write_space(struct sock *sk);
unsigned int sctp_poll(struct file *file, struct socket *sock,
poll_table *wait);
void sctp_sock_rfree(struct sk_buff *skb);
extern struct percpu_counter sctp_sockets_allocated;

/*
* sctp/primitive.c
2 changes: 1 addition & 1 deletion include/net/sock.h
Original file line number Diff line number Diff line change
@@ -649,7 +649,7 @@ struct proto {
/* Memory pressure */
void (*enter_memory_pressure)(struct sock *sk);
atomic_t *memory_allocated; /* Current allocated memory. */
atomic_t *sockets_allocated; /* Current number of sockets. */
struct percpu_counter *sockets_allocated; /* Current number of sockets. */
/*
* Pressure flag: try to collapse.
* Technical note: it is used by multiple contexts non atomically.
2 changes: 1 addition & 1 deletion include/net/tcp.h
Original file line number Diff line number Diff line change
@@ -238,7 +238,7 @@ extern int sysctl_tcp_slow_start_after_idle;
extern int sysctl_tcp_max_ssthresh;

extern atomic_t tcp_memory_allocated;
extern atomic_t tcp_sockets_allocated;
extern struct percpu_counter tcp_sockets_allocated;
extern int tcp_memory_pressure;

/*
10 changes: 7 additions & 3 deletions net/core/sock.c
Original file line number Diff line number Diff line change
@@ -1071,7 +1071,7 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority)
newsk->sk_sleep = NULL;

if (newsk->sk_prot->sockets_allocated)
atomic_inc(newsk->sk_prot->sockets_allocated);
percpu_counter_inc(newsk->sk_prot->sockets_allocated);
}
out:
return newsk;
@@ -1463,8 +1463,12 @@ int __sk_mem_schedule(struct sock *sk, int size, int kind)
}

if (prot->memory_pressure) {
if (!*prot->memory_pressure ||
prot->sysctl_mem[2] > atomic_read(prot->sockets_allocated) *
int alloc;

if (!*prot->memory_pressure)
return 1;
alloc = percpu_counter_read_positive(prot->sockets_allocated);
if (prot->sysctl_mem[2] > alloc *
sk_mem_pages(sk->sk_wmem_queued +
atomic_read(&sk->sk_rmem_alloc) +
sk->sk_forward_alloc))
3 changes: 2 additions & 1 deletion net/ipv4/proc.c
Original file line number Diff line number Diff line change
@@ -55,7 +55,8 @@ static int sockstat_seq_show(struct seq_file *seq, void *v)
seq_printf(seq, "TCP: inuse %d orphan %d tw %d alloc %d mem %d\n",
sock_prot_inuse_get(net, &tcp_prot),
atomic_read(&tcp_orphan_count),
tcp_death_row.tw_count, atomic_read(&tcp_sockets_allocated),
tcp_death_row.tw_count,
(int)percpu_counter_sum_positive(&tcp_sockets_allocated),
atomic_read(&tcp_memory_allocated));
seq_printf(seq, "UDP: inuse %d mem %d\n",
sock_prot_inuse_get(net, &udp_prot),
8 changes: 6 additions & 2 deletions net/ipv4/tcp.c
Original file line number Diff line number Diff line change
@@ -290,9 +290,12 @@ EXPORT_SYMBOL(sysctl_tcp_rmem);
EXPORT_SYMBOL(sysctl_tcp_wmem);

atomic_t tcp_memory_allocated; /* Current allocated memory. */
atomic_t tcp_sockets_allocated; /* Current number of TCP sockets. */

EXPORT_SYMBOL(tcp_memory_allocated);

/*
* Current number of TCP sockets.
*/
struct percpu_counter tcp_sockets_allocated;
EXPORT_SYMBOL(tcp_sockets_allocated);

/*
@@ -2685,6 +2688,7 @@ void __init tcp_init(void)

BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > sizeof(skb->cb));

percpu_counter_init(&tcp_sockets_allocated, 0);
tcp_hashinfo.bind_bucket_cachep =
kmem_cache_create("tcp_bind_bucket",
sizeof(struct inet_bind_bucket), 0,
4 changes: 2 additions & 2 deletions net/ipv4/tcp_ipv4.c
Original file line number Diff line number Diff line change
@@ -1797,7 +1797,7 @@ static int tcp_v4_init_sock(struct sock *sk)
sk->sk_sndbuf = sysctl_tcp_wmem[1];
sk->sk_rcvbuf = sysctl_tcp_rmem[1];

atomic_inc(&tcp_sockets_allocated);
percpu_counter_inc(&tcp_sockets_allocated);

return 0;
}
@@ -1845,7 +1845,7 @@ void tcp_v4_destroy_sock(struct sock *sk)
sk->sk_sndmsg_page = NULL;
}

atomic_dec(&tcp_sockets_allocated);
percpu_counter_dec(&tcp_sockets_allocated);
}

EXPORT_SYMBOL(tcp_v4_destroy_sock);
2 changes: 1 addition & 1 deletion net/ipv6/tcp_ipv6.c
Original file line number Diff line number Diff line change
@@ -1830,7 +1830,7 @@ static int tcp_v6_init_sock(struct sock *sk)
sk->sk_sndbuf = sysctl_tcp_wmem[1];
sk->sk_rcvbuf = sysctl_tcp_rmem[1];

atomic_inc(&tcp_sockets_allocated);
percpu_counter_inc(&tcp_sockets_allocated);

return 0;
}
6 changes: 5 additions & 1 deletion net/sctp/protocol.c
Original file line number Diff line number Diff line change
@@ -102,6 +102,8 @@ struct sock *sctp_get_ctl_sock(void)
/* Set up the proc fs entry for the SCTP protocol. */
static __init int sctp_proc_init(void)
{
if (percpu_counter_init(&sctp_sockets_allocated, 0))
goto out_nomem;
#ifdef CONFIG_PROC_FS
if (!proc_net_sctp) {
struct proc_dir_entry *ent;
@@ -110,7 +112,7 @@ static __init int sctp_proc_init(void)
ent->owner = THIS_MODULE;
proc_net_sctp = ent;
} else
goto out_nomem;
goto out_free_percpu;
}

if (sctp_snmp_proc_init())
@@ -135,6 +137,8 @@ static __init int sctp_proc_init(void)
proc_net_sctp = NULL;
remove_proc_entry("sctp", init_net.proc_net);
}
out_free_percpu:
percpu_counter_destroy(&sctp_sockets_allocated);
out_nomem:
return -ENOMEM;
#else
6 changes: 3 additions & 3 deletions net/sctp/socket.c
Original file line number Diff line number Diff line change
@@ -114,7 +114,7 @@ extern int sysctl_sctp_wmem[3];

static int sctp_memory_pressure;
static atomic_t sctp_memory_allocated;
static atomic_t sctp_sockets_allocated;
struct percpu_counter sctp_sockets_allocated;

static void sctp_enter_memory_pressure(struct sock *sk)
{
@@ -3613,7 +3613,7 @@ SCTP_STATIC int sctp_init_sock(struct sock *sk)
sp->hmac = NULL;

SCTP_DBG_OBJCNT_INC(sock);
atomic_inc(&sctp_sockets_allocated);
percpu_counter_inc(&sctp_sockets_allocated);

local_bh_disable();
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
@@ -3632,7 +3632,7 @@ SCTP_STATIC void sctp_destroy_sock(struct sock *sk)
/* Release our hold on the endpoint. */
ep = sctp_sk(sk)->ep;
sctp_endpoint_free(ep);
atomic_dec(&sctp_sockets_allocated);
percpu_counter_dec(&sctp_sockets_allocated);
local_bh_disable();
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
local_bh_enable();
1 change: 0 additions & 1 deletion net/unix/af_unix.c
Original file line number Diff line number Diff line change
@@ -571,7 +571,6 @@ static const struct proto_ops unix_seqpacket_ops = {
static struct proto unix_proto = {
.name = "UNIX",
.owner = THIS_MODULE,
.sockets_allocated = &unix_nr_socks,
.obj_size = sizeof(struct unix_sock),
};

0 comments on commit 1748376

Please sign in to comment.