Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history 
yaml
---
r: 121524
b: refs/heads/master
c: 271b72c
h: refs/heads/master
v: v3
  • Loading branch information
Eric Dumazet authored and David S. Miller committed Oct 29, 2008
1 parent 78c7a7c commit 46d851c
Show file tree
Hide file tree
Showing 7 changed files with 91 additions and 19 deletions.
2 changes: 1 addition & 1 deletion [refs]
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: 645ca708f936b2fbeb79e52d7823e3eb2c0905f8
refs/heads/master: 271b72c7fa82c2c7a795bc16896149933110672d
37 changes: 36 additions & 1 deletion trunk/include/net/sock.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,27 @@ static __inline__ int sk_del_node_init(struct sock *sk)
return rc;
}

static __inline__ int __sk_del_node_init_rcu(struct sock *sk)
{
if (sk_hashed(sk)) {
hlist_del_init_rcu(&sk->sk_node);
return 1;
}
return 0;
}

static __inline__ int sk_del_node_init_rcu(struct sock *sk)
{
int rc = __sk_del_node_init_rcu(sk);

if (rc) {
/* paranoid for a while -acme */
WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
__sock_put(sk);
}
return rc;
}

static __inline__ void __sk_add_node(struct sock *sk, struct hlist_head *list)
{
hlist_add_head(&sk->sk_node, list);
Expand All @@ -374,6 +395,17 @@ static __inline__ void sk_add_node(struct sock *sk, struct hlist_head *list)
__sk_add_node(sk, list);
}

static __inline__ void __sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
{
hlist_add_head_rcu(&sk->sk_node, list);
}

static __inline__ void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
{
sock_hold(sk);
__sk_add_node_rcu(sk, list);
}

static __inline__ void __sk_del_bind_node(struct sock *sk)
{
__hlist_del(&sk->sk_bind_node);
Expand All @@ -387,6 +419,8 @@ static __inline__ void sk_add_bind_node(struct sock *sk,

#define sk_for_each(__sk, node, list) \
hlist_for_each_entry(__sk, node, list, sk_node)
#define sk_for_each_rcu(__sk, node, list) \
hlist_for_each_entry_rcu(__sk, node, list, sk_node)
#define sk_for_each_from(__sk, node) \
if (__sk && ({ node = &(__sk)->sk_node; 1; })) \
hlist_for_each_entry_from(__sk, node, sk_node)
Expand Down Expand Up @@ -589,8 +623,9 @@ struct proto {
int *sysctl_rmem;
int max_header;

struct kmem_cache *slab;
struct kmem_cache *slab;
unsigned int obj_size;
int slab_flags;

atomic_t *orphan_count;

Expand Down
3 changes: 2 additions & 1 deletion trunk/net/core/sock.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2042,7 +2042,8 @@ int proto_register(struct proto *prot, int alloc_slab)

if (alloc_slab) {
prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0,
SLAB_HWCACHE_ALIGN, NULL);
SLAB_HWCACHE_ALIGN | prot->slab_flags,
NULL);

if (prot->slab == NULL) {
printk(KERN_CRIT "%s: Can't create sock SLAB cache!\n",
Expand Down
35 changes: 26 additions & 9 deletions trunk/net/ipv4/udp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum,
inet_sk(sk)->num = snum;
sk->sk_hash = snum;
if (sk_unhashed(sk)) {
sk_add_node(sk, &hslot->head);
sk_add_node_rcu(sk, &hslot->head);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
}
error = 0;
Expand Down Expand Up @@ -253,25 +253,41 @@ static struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
__be16 sport, __be32 daddr, __be16 dport,
int dif, struct udp_table *udptable)
{
struct sock *sk, *result = NULL;
struct sock *sk, *result;
struct hlist_node *node;
unsigned short hnum = ntohs(dport);
unsigned int hash = udp_hashfn(net, hnum);
struct udp_hslot *hslot = &udptable->hash[hash];
int score, badness = -1;
int score, badness;

spin_lock(&hslot->lock);
sk_for_each(sk, node, &hslot->head) {
rcu_read_lock();
begin:
result = NULL;
badness = -1;
sk_for_each_rcu(sk, node, &hslot->head) {
/*
* lockless reader, and SLAB_DESTROY_BY_RCU items:
* We must check this item was not moved to another chain
*/
if (udp_hashfn(net, sk->sk_hash) != hash)
goto begin;
score = compute_score(sk, net, saddr, hnum, sport,
daddr, dport, dif);
if (score > badness) {
result = sk;
badness = score;
}
}
if (result)
sock_hold(result);
spin_unlock(&hslot->lock);
if (result) {
if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
result = NULL;
else if (unlikely(compute_score(result, net, saddr, hnum, sport,
daddr, dport, dif) < badness)) {
sock_put(result);
goto begin;
}
}
rcu_read_unlock();
return result;
}

Expand Down Expand Up @@ -953,7 +969,7 @@ void udp_lib_unhash(struct sock *sk)
struct udp_hslot *hslot = &udptable->hash[hash];

spin_lock(&hslot->lock);
if (sk_del_node_init(sk)) {
if (sk_del_node_init_rcu(sk)) {
inet_sk(sk)->num = 0;
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
}
Expand Down Expand Up @@ -1517,6 +1533,7 @@ struct proto udp_prot = {
.sysctl_wmem = &sysctl_udp_wmem_min,
.sysctl_rmem = &sysctl_udp_rmem_min,
.obj_size = sizeof(struct udp_sock),
.slab_flags = SLAB_DESTROY_BY_RCU,
.h.udp_table = &udp_table,
#ifdef CONFIG_COMPAT
.compat_setsockopt = compat_udp_setsockopt,
Expand Down
1 change: 1 addition & 0 deletions trunk/net/ipv4/udplite.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ struct proto udplite_prot = {
.unhash = udp_lib_unhash,
.get_port = udp_v4_get_port,
.obj_size = sizeof(struct udp_sock),
.slab_flags = SLAB_DESTROY_BY_RCU,
.h.udp_table = &udplite_table,
#ifdef CONFIG_COMPAT
.compat_setsockopt = compat_udp_setsockopt,
Expand Down
31 changes: 24 additions & 7 deletions trunk/net/ipv6/udp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,24 +97,40 @@ static struct sock *__udp6_lib_lookup(struct net *net,
struct in6_addr *daddr, __be16 dport,
int dif, struct udp_table *udptable)
{
struct sock *sk, *result = NULL;
struct sock *sk, *result;
struct hlist_node *node;
unsigned short hnum = ntohs(dport);
unsigned int hash = udp_hashfn(net, hnum);
struct udp_hslot *hslot = &udptable->hash[hash];
int score, badness = -1;
int score, badness;

spin_lock(&hslot->lock);
sk_for_each(sk, node, &hslot->head) {
rcu_read_lock();
begin:
result = NULL;
badness = -1;
sk_for_each_rcu(sk, node, &hslot->head) {
/*
* lockless reader, and SLAB_DESTROY_BY_RCU items:
* We must check this item was not moved to another chain
*/
if (udp_hashfn(net, sk->sk_hash) != hash)
goto begin;
score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif);
if (score > badness) {
result = sk;
badness = score;
}
}
if (result)
sock_hold(result);
spin_unlock(&hslot->lock);
if (result) {
if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
result = NULL;
else if (unlikely(compute_score(result, net, hnum, saddr, sport,
daddr, dport, dif) < badness)) {
sock_put(result);
goto begin;
}
}
rcu_read_unlock();
return result;
}

Expand Down Expand Up @@ -1062,6 +1078,7 @@ struct proto udpv6_prot = {
.sysctl_wmem = &sysctl_udp_wmem_min,
.sysctl_rmem = &sysctl_udp_rmem_min,
.obj_size = sizeof(struct udp6_sock),
.slab_flags = SLAB_DESTROY_BY_RCU,
.h.udp_table = &udp_table,
#ifdef CONFIG_COMPAT
.compat_setsockopt = compat_udpv6_setsockopt,
Expand Down
1 change: 1 addition & 0 deletions trunk/net/ipv6/udplite.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ struct proto udplitev6_prot = {
.unhash = udp_lib_unhash,
.get_port = udp_v6_get_port,
.obj_size = sizeof(struct udp6_sock),
.slab_flags = SLAB_DESTROY_BY_RCU,
.h.udp_table = &udplite_table,
#ifdef CONFIG_COMPAT
.compat_setsockopt = compat_udpv6_setsockopt,
Expand Down

0 comments on commit 46d851c

Please sign in to comment.