Skip to content

Commit

Permalink
ipvs: do not disable bh for long time
Browse files Browse the repository at this point in the history 
We used a global BH disable in LOCAL_OUT hook.
Add _bh suffix to all places that need it and remove
the disabling from LOCAL_OUT and sync code.

Functions like ip_defrag need protection from
BH, so add it. As for nf_nat_mangle_tcp_packet, it needs
RCU lock.

Signed-off-by: Julian Anastasov <ja@ssi.bg>
Signed-off-by: Simon Horman <horms@verge.net.au>
  • Loading branch information
Julian Anastasov authored and Pablo Neira Ayuso committed Apr 1, 2013
1 parent ceec4c3 commit ac69269
Show file tree
Hide file tree
Showing 12 changed files with 64 additions and 87 deletions.
4 changes: 2 additions & 2 deletions net/netfilter/ipvs/ip_vs_app.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,14 +352,14 @@ static inline void vs_seq_update(struct ip_vs_conn *cp, struct ip_vs_seq *vseq,
unsigned int flag, __u32 seq, int diff)
{
/* spinlock is to keep updating cp->flags atomic */
spin_lock(&cp->lock);
spin_lock_bh(&cp->lock);
if (!(cp->flags & flag) || after(seq, vseq->init_seq)) {
vseq->previous_delta = vseq->delta;
vseq->delta += diff;
vseq->init_seq = seq;
cp->flags |= flag;
}
spin_unlock(&cp->lock);
spin_unlock_bh(&cp->lock);
}

static inline int app_tcp_pkt_out(struct ip_vs_conn *cp, struct sk_buff *skb,
Expand Down
30 changes: 15 additions & 15 deletions net/netfilter/ipvs/ip_vs_conn.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,14 @@ struct ip_vs_aligned_lock
static struct ip_vs_aligned_lock
__ip_vs_conntbl_lock_array[CT_LOCKARRAY_SIZE] __cacheline_aligned;

static inline void ct_write_lock(unsigned int key)
static inline void ct_write_lock_bh(unsigned int key)
{
spin_lock(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
spin_lock_bh(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
}

static inline void ct_write_unlock(unsigned int key)
static inline void ct_write_unlock_bh(unsigned int key)
{
spin_unlock(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
spin_unlock_bh(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
}


Expand Down Expand Up @@ -167,7 +167,7 @@ static inline int ip_vs_conn_hash(struct ip_vs_conn *cp)
/* Hash by protocol, client address and port */
hash = ip_vs_conn_hashkey_conn(cp);

ct_write_lock(hash);
ct_write_lock_bh(hash);
spin_lock(&cp->lock);

if (!(cp->flags & IP_VS_CONN_F_HASHED)) {
Expand All @@ -182,7 +182,7 @@ static inline int ip_vs_conn_hash(struct ip_vs_conn *cp)
}

spin_unlock(&cp->lock);
ct_write_unlock(hash);
ct_write_unlock_bh(hash);

return ret;
}
Expand All @@ -200,7 +200,7 @@ static inline int ip_vs_conn_unhash(struct ip_vs_conn *cp)
/* unhash it and decrease its reference counter */
hash = ip_vs_conn_hashkey_conn(cp);

ct_write_lock(hash);
ct_write_lock_bh(hash);
spin_lock(&cp->lock);

if (cp->flags & IP_VS_CONN_F_HASHED) {
Expand All @@ -212,7 +212,7 @@ static inline int ip_vs_conn_unhash(struct ip_vs_conn *cp)
ret = 0;

spin_unlock(&cp->lock);
ct_write_unlock(hash);
ct_write_unlock_bh(hash);

return ret;
}
Expand All @@ -227,7 +227,7 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp)

hash = ip_vs_conn_hashkey_conn(cp);

ct_write_lock(hash);
ct_write_lock_bh(hash);
spin_lock(&cp->lock);

if (cp->flags & IP_VS_CONN_F_HASHED) {
Expand All @@ -242,7 +242,7 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp)
ret = atomic_read(&cp->refcnt) ? false : true;

spin_unlock(&cp->lock);
ct_write_unlock(hash);
ct_write_unlock_bh(hash);

return ret;
}
Expand Down Expand Up @@ -462,13 +462,13 @@ void ip_vs_conn_put(struct ip_vs_conn *cp)
void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport)
{
if (ip_vs_conn_unhash(cp)) {
spin_lock(&cp->lock);
spin_lock_bh(&cp->lock);
if (cp->flags & IP_VS_CONN_F_NO_CPORT) {
atomic_dec(&ip_vs_conn_no_cport_cnt);
cp->flags &= ~IP_VS_CONN_F_NO_CPORT;
cp->cport = cport;
}
spin_unlock(&cp->lock);
spin_unlock_bh(&cp->lock);

/* hash on new dport */
ip_vs_conn_hash(cp);
Expand Down Expand Up @@ -622,9 +622,9 @@ void ip_vs_try_bind_dest(struct ip_vs_conn *cp)
if (dest) {
struct ip_vs_proto_data *pd;

spin_lock(&cp->lock);
spin_lock_bh(&cp->lock);
if (cp->dest) {
spin_unlock(&cp->lock);
spin_unlock_bh(&cp->lock);
rcu_read_unlock();
return;
}
Expand All @@ -635,7 +635,7 @@ void ip_vs_try_bind_dest(struct ip_vs_conn *cp)
ip_vs_unbind_app(cp);

ip_vs_bind_dest(cp, dest);
spin_unlock(&cp->lock);
spin_unlock_bh(&cp->lock);

/* Update its packet transmitter */
cp->packet_xmit = NULL;
Expand Down
37 changes: 8 additions & 29 deletions net/netfilter/ipvs/ip_vs_core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -638,8 +638,11 @@ static inline enum ip_defrag_users ip_vs_defrag_user(unsigned int hooknum)

static inline int ip_vs_gather_frags(struct sk_buff *skb, u_int32_t user)
{
int err = ip_defrag(skb, user);
int err;

local_bh_disable();
err = ip_defrag(skb, user);
local_bh_enable();
if (!err)
ip_send_check(ip_hdr(skb));

Expand Down Expand Up @@ -1217,13 +1220,7 @@ ip_vs_local_reply4(unsigned int hooknum, struct sk_buff *skb,
const struct net_device *in, const struct net_device *out,
int (*okfn)(struct sk_buff *))
{
unsigned int verdict;

/* Disable BH in LOCAL_OUT until all places are fixed */
local_bh_disable();
verdict = ip_vs_out(hooknum, skb, AF_INET);
local_bh_enable();
return verdict;
return ip_vs_out(hooknum, skb, AF_INET);
}

#ifdef CONFIG_IP_VS_IPV6
Expand All @@ -1250,13 +1247,7 @@ ip_vs_local_reply6(unsigned int hooknum, struct sk_buff *skb,
const struct net_device *in, const struct net_device *out,
int (*okfn)(struct sk_buff *))
{
unsigned int verdict;

/* Disable BH in LOCAL_OUT until all places are fixed */
local_bh_disable();
verdict = ip_vs_out(hooknum, skb, AF_INET6);
local_bh_enable();
return verdict;
return ip_vs_out(hooknum, skb, AF_INET6);
}

#endif
Expand Down Expand Up @@ -1714,13 +1705,7 @@ ip_vs_local_request4(unsigned int hooknum, struct sk_buff *skb,
const struct net_device *in, const struct net_device *out,
int (*okfn)(struct sk_buff *))
{
unsigned int verdict;

/* Disable BH in LOCAL_OUT until all places are fixed */
local_bh_disable();
verdict = ip_vs_in(hooknum, skb, AF_INET);
local_bh_enable();
return verdict;
return ip_vs_in(hooknum, skb, AF_INET);
}

#ifdef CONFIG_IP_VS_IPV6
Expand Down Expand Up @@ -1779,13 +1764,7 @@ ip_vs_local_request6(unsigned int hooknum, struct sk_buff *skb,
const struct net_device *in, const struct net_device *out,
int (*okfn)(struct sk_buff *))
{
unsigned int verdict;

/* Disable BH in LOCAL_OUT until all places are fixed */
local_bh_disable();
verdict = ip_vs_in(hooknum, skb, AF_INET6);
local_bh_enable();
return verdict;
return ip_vs_in(hooknum, skb, AF_INET6);
}

#endif
Expand Down
2 changes: 2 additions & 0 deletions net/netfilter/ipvs/ip_vs_ftp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,10 +267,12 @@ static int ip_vs_ftp_out(struct ip_vs_app *app, struct ip_vs_conn *cp,
* hopefully it will succeed on the retransmitted
* packet.
*/
rcu_read_lock();
ret = nf_nat_mangle_tcp_packet(skb, ct, ctinfo,
iph->ihl * 4,
start-data, end-start,
buf, buf_len);
rcu_read_unlock();
if (ret) {
ip_vs_nfct_expect_related(skb, ct, n_cp,
IPPROTO_TCP, 0, 0);
Expand Down
4 changes: 2 additions & 2 deletions net/netfilter/ipvs/ip_vs_lblc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -527,10 +527,10 @@ ip_vs_lblc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
}

/* If we fail to create a cache entry, we'll just use the valid dest */
spin_lock(&svc->sched_lock);
spin_lock_bh(&svc->sched_lock);
if (!tbl->dead)
ip_vs_lblc_new(tbl, &iph.daddr, dest);
spin_unlock(&svc->sched_lock);
spin_unlock_bh(&svc->sched_lock);

out:
IP_VS_DBG_BUF(6, "LBLC: destination IP address %s --> server %s:%d\n",
Expand Down
12 changes: 6 additions & 6 deletions net/netfilter/ipvs/ip_vs_lblcr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -678,15 +678,15 @@ ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
if (atomic_read(&en->set.size) > 1 &&
time_after(jiffies, en->set.lastmod +
sysctl_lblcr_expiration(svc))) {
spin_lock(&svc->sched_lock);
spin_lock_bh(&svc->sched_lock);
if (atomic_read(&en->set.size) > 1) {
struct ip_vs_dest *m;

m = ip_vs_dest_set_max(&en->set);
if (m)
ip_vs_dest_set_erase(&en->set, m);
}
spin_unlock(&svc->sched_lock);
spin_unlock_bh(&svc->sched_lock);
}

/* If the destination is not overloaded, use it */
Expand All @@ -701,10 +701,10 @@ ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
}

/* Update our cache entry */
spin_lock(&svc->sched_lock);
spin_lock_bh(&svc->sched_lock);
if (!tbl->dead)
ip_vs_dest_set_insert(&en->set, dest, true);
spin_unlock(&svc->sched_lock);
spin_unlock_bh(&svc->sched_lock);
goto out;
}

Expand All @@ -716,10 +716,10 @@ ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
}

/* If we fail to create a cache entry, we'll just use the valid dest */
spin_lock(&svc->sched_lock);
spin_lock_bh(&svc->sched_lock);
if (!tbl->dead)
ip_vs_lblcr_new(tbl, &iph.daddr, dest);
spin_unlock(&svc->sched_lock);
spin_unlock_bh(&svc->sched_lock);

out:
IP_VS_DBG_BUF(6, "LBLCR: destination IP address %s --> server %s:%d\n",
Expand Down
4 changes: 2 additions & 2 deletions net/netfilter/ipvs/ip_vs_proto_sctp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -994,9 +994,9 @@ static void
sctp_state_transition(struct ip_vs_conn *cp, int direction,
const struct sk_buff *skb, struct ip_vs_proto_data *pd)
{
spin_lock(&cp->lock);
spin_lock_bh(&cp->lock);
set_sctp_state(pd, cp, direction, skb);
spin_unlock(&cp->lock);
spin_unlock_bh(&cp->lock);
}

static inline __u16 sctp_app_hashkey(__be16 port)
Expand Down
8 changes: 4 additions & 4 deletions net/netfilter/ipvs/ip_vs_proto_tcp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -557,9 +557,9 @@ tcp_state_transition(struct ip_vs_conn *cp, int direction,
if (th == NULL)
return;

spin_lock(&cp->lock);
spin_lock_bh(&cp->lock);
set_tcp_state(pd, cp, direction, th);
spin_unlock(&cp->lock);
spin_unlock_bh(&cp->lock);
}

static inline __u16 tcp_app_hashkey(__be16 port)
Expand Down Expand Up @@ -655,11 +655,11 @@ void ip_vs_tcp_conn_listen(struct net *net, struct ip_vs_conn *cp)
{
struct ip_vs_proto_data *pd = ip_vs_proto_data_get(net, IPPROTO_TCP);

spin_lock(&cp->lock);
spin_lock_bh(&cp->lock);
cp->state = IP_VS_TCP_S_LISTEN;
cp->timeout = (pd ? pd->timeout_table[IP_VS_TCP_S_LISTEN]
: tcp_timeouts[IP_VS_TCP_S_LISTEN]);
spin_unlock(&cp->lock);
spin_unlock_bh(&cp->lock);
}

/* ---------------------------------------------
Expand Down
6 changes: 3 additions & 3 deletions net/netfilter/ipvs/ip_vs_rr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ ip_vs_rr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)

IP_VS_DBG(6, "%s(): Scheduling...\n", __func__);

spin_lock(&svc->sched_lock);
spin_lock_bh(&svc->sched_lock);
p = (struct list_head *) svc->sched_data;
last = dest = list_entry(p, struct ip_vs_dest, n_list);

Expand All @@ -85,13 +85,13 @@ ip_vs_rr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
} while (pass < 2 && p != &svc->destinations);

stop:
spin_unlock(&svc->sched_lock);
spin_unlock_bh(&svc->sched_lock);
ip_vs_scheduler_err(svc, "no destination available");
return NULL;

out:
svc->sched_data = &dest->n_list;
spin_unlock(&svc->sched_lock);
spin_unlock_bh(&svc->sched_lock);
IP_VS_DBG_BUF(6, "RR: server %s:%u "
"activeconns %d refcnt %d weight %d\n",
IP_VS_DBG_ADDR(svc->af, &dest->addr), ntohs(dest->port),
Expand Down
Loading

0 comments on commit ac69269

Please sign in to comment.