Skip to content

Commit

Permalink
netfilter: nf_conntrack: use SLAB_DESTROY_BY_RCU and get rid of call_…
Browse files Browse the repository at this point in the history
…rcu()

Use "hlist_nulls" infrastructure we added in 2.6.29 for RCUification of UDP & TCP.

This permits an easy conversion from call_rcu() based hash lists to a
SLAB_DESTROY_BY_RCU one.

Avoiding call_rcu() delay at nf_conn freeing time has numerous gains.

First, it doesnt fill RCU queues (up to 10000 elements per cpu).
This reduces OOM possibility, if queued elements are not taken into account
This reduces latency problems when RCU queue size hits hilimit and triggers
emergency mode.

- It allows fast reuse of just freed elements, permitting better use of
CPU cache.

- We delete rcu_head from "struct nf_conn", shrinking size of this structure
by 8 or 16 bytes.

This patch only takes care of "struct nf_conn".
call_rcu() is still used for less critical conntrack parts, that may
be converted later if necessary.

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: Patrick McHardy <kaber@trash.net>
  • Loading branch information
Eric Dumazet authored and Patrick McHardy committed Mar 25, 2009
1 parent 1f9352a commit ea781f1
Show file tree
Hide file tree
Showing 11 changed files with 174 additions and 131 deletions.
14 changes: 8 additions & 6 deletions include/net/netfilter/nf_conntrack.h
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,7 @@ struct nf_conn_help {
#include <net/netfilter/ipv4/nf_conntrack_ipv4.h>
#include <net/netfilter/ipv6/nf_conntrack_ipv6.h>

struct nf_conn
{
struct nf_conn {
/* Usage count in here is 1 for hash table/destruct timer, 1 per skb,
plus 1 for any connection(s) we are `master' for */
struct nf_conntrack ct_general;
Expand Down Expand Up @@ -126,7 +125,6 @@ struct nf_conn
#ifdef CONFIG_NET_NS
struct net *ct_net;
#endif
struct rcu_head rcu;
};

static inline struct nf_conn *
Expand Down Expand Up @@ -190,9 +188,13 @@ static inline void nf_ct_put(struct nf_conn *ct)
extern int nf_ct_l3proto_try_module_get(unsigned short l3proto);
extern void nf_ct_l3proto_module_put(unsigned short l3proto);

extern struct hlist_head *nf_ct_alloc_hashtable(unsigned int *sizep, int *vmalloced);
extern void nf_ct_free_hashtable(struct hlist_head *hash, int vmalloced,
unsigned int size);
/*
* Allocate a hashtable of hlist_head (if nulls == 0),
* or hlist_nulls_head (if nulls == 1)
*/
extern void *nf_ct_alloc_hashtable(unsigned int *sizep, int *vmalloced, int nulls);

extern void nf_ct_free_hashtable(void *hash, int vmalloced, unsigned int size);

extern struct nf_conntrack_tuple_hash *
__nf_conntrack_find(struct net *net, const struct nf_conntrack_tuple *tuple);
Expand Down
6 changes: 3 additions & 3 deletions include/net/netfilter/nf_conntrack_tuple.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

#include <linux/netfilter/x_tables.h>
#include <linux/netfilter/nf_conntrack_tuple_common.h>
#include <linux/list_nulls.h>

/* A `tuple' is a structure containing the information to uniquely
identify a connection. ie. if two packets have the same tuple, they
Expand Down Expand Up @@ -146,9 +147,8 @@ static inline void nf_ct_dump_tuple(const struct nf_conntrack_tuple *t)
((enum ip_conntrack_dir)(h)->tuple.dst.dir)

/* Connections have two entries in the hash table: one for each way */
struct nf_conntrack_tuple_hash
{
struct hlist_node hnode;
struct nf_conntrack_tuple_hash {
struct hlist_nulls_node hnnode;
struct nf_conntrack_tuple tuple;
};

Expand Down
5 changes: 3 additions & 2 deletions include/net/netns/conntrack.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#define __NETNS_CONNTRACK_H

#include <linux/list.h>
#include <linux/list_nulls.h>
#include <asm/atomic.h>

struct ctl_table_header;
Expand All @@ -10,9 +11,9 @@ struct nf_conntrack_ecache;
struct netns_ct {
atomic_t count;
unsigned int expect_count;
struct hlist_head *hash;
struct hlist_nulls_head *hash;
struct hlist_head *expect_hash;
struct hlist_head unconfirmed;
struct hlist_nulls_head unconfirmed;
struct ip_conntrack_stat *stat;
#ifdef CONFIG_NF_CONNTRACK_EVENTS
struct nf_conntrack_ecache *ecache;
Expand Down
63 changes: 36 additions & 27 deletions net/ipv4/netfilter/nf_conntrack_l3proto_ipv4_compat.c
Original file line number Diff line number Diff line change
Expand Up @@ -25,40 +25,42 @@ struct ct_iter_state {
unsigned int bucket;
};

static struct hlist_node *ct_get_first(struct seq_file *seq)
static struct hlist_nulls_node *ct_get_first(struct seq_file *seq)
{
struct net *net = seq_file_net(seq);
struct ct_iter_state *st = seq->private;
struct hlist_node *n;
struct hlist_nulls_node *n;

for (st->bucket = 0;
st->bucket < nf_conntrack_htable_size;
st->bucket++) {
n = rcu_dereference(net->ct.hash[st->bucket].first);
if (n)
if (!is_a_nulls(n))
return n;
}
return NULL;
}

static struct hlist_node *ct_get_next(struct seq_file *seq,
struct hlist_node *head)
static struct hlist_nulls_node *ct_get_next(struct seq_file *seq,
struct hlist_nulls_node *head)
{
struct net *net = seq_file_net(seq);
struct ct_iter_state *st = seq->private;

head = rcu_dereference(head->next);
while (head == NULL) {
if (++st->bucket >= nf_conntrack_htable_size)
return NULL;
while (is_a_nulls(head)) {
if (likely(get_nulls_value(head) == st->bucket)) {
if (++st->bucket >= nf_conntrack_htable_size)
return NULL;
}
head = rcu_dereference(net->ct.hash[st->bucket].first);
}
return head;
}

static struct hlist_node *ct_get_idx(struct seq_file *seq, loff_t pos)
static struct hlist_nulls_node *ct_get_idx(struct seq_file *seq, loff_t pos)
{
struct hlist_node *head = ct_get_first(seq);
struct hlist_nulls_node *head = ct_get_first(seq);

if (head)
while (pos && (head = ct_get_next(seq, head)))
Expand Down Expand Up @@ -87,69 +89,76 @@ static void ct_seq_stop(struct seq_file *s, void *v)

static int ct_seq_show(struct seq_file *s, void *v)
{
const struct nf_conntrack_tuple_hash *hash = v;
const struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(hash);
struct nf_conntrack_tuple_hash *hash = v;
struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(hash);
const struct nf_conntrack_l3proto *l3proto;
const struct nf_conntrack_l4proto *l4proto;
int ret = 0;

NF_CT_ASSERT(ct);
if (unlikely(!atomic_inc_not_zero(&ct->ct_general.use)))
return 0;


/* we only want to print DIR_ORIGINAL */
if (NF_CT_DIRECTION(hash))
return 0;
goto release;
if (nf_ct_l3num(ct) != AF_INET)
return 0;
goto release;

l3proto = __nf_ct_l3proto_find(nf_ct_l3num(ct));
NF_CT_ASSERT(l3proto);
l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
NF_CT_ASSERT(l4proto);

ret = -ENOSPC;
if (seq_printf(s, "%-8s %u %ld ",
l4proto->name, nf_ct_protonum(ct),
timer_pending(&ct->timeout)
? (long)(ct->timeout.expires - jiffies)/HZ : 0) != 0)
return -ENOSPC;
goto release;

if (l4proto->print_conntrack && l4proto->print_conntrack(s, ct))
return -ENOSPC;
goto release;

if (print_tuple(s, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
l3proto, l4proto))
return -ENOSPC;
goto release;

if (seq_print_acct(s, ct, IP_CT_DIR_ORIGINAL))
return -ENOSPC;
goto release;

if (!(test_bit(IPS_SEEN_REPLY_BIT, &ct->status)))
if (seq_printf(s, "[UNREPLIED] "))
return -ENOSPC;
goto release;

if (print_tuple(s, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
l3proto, l4proto))
return -ENOSPC;
goto release;

if (seq_print_acct(s, ct, IP_CT_DIR_REPLY))
return -ENOSPC;
goto release;

if (test_bit(IPS_ASSURED_BIT, &ct->status))
if (seq_printf(s, "[ASSURED] "))
return -ENOSPC;
goto release;

#ifdef CONFIG_NF_CONNTRACK_MARK
if (seq_printf(s, "mark=%u ", ct->mark))
return -ENOSPC;
goto release;
#endif

#ifdef CONFIG_NF_CONNTRACK_SECMARK
if (seq_printf(s, "secmark=%u ", ct->secmark))
return -ENOSPC;
goto release;
#endif

if (seq_printf(s, "use=%u\n", atomic_read(&ct->ct_general.use)))
return -ENOSPC;

return 0;
goto release;
ret = 0;
release:
nf_ct_put(ct);
return ret;
}

static const struct seq_operations ct_seq_ops = {
Expand Down
2 changes: 1 addition & 1 deletion net/ipv4/netfilter/nf_nat_core.c
Original file line number Diff line number Diff line change
Expand Up @@ -679,7 +679,7 @@ nfnetlink_parse_nat_setup(struct nf_conn *ct,
static int __net_init nf_nat_net_init(struct net *net)
{
net->ipv4.nat_bysource = nf_ct_alloc_hashtable(&nf_nat_htable_size,
&net->ipv4.nat_vmalloced);
&net->ipv4.nat_vmalloced, 0);
if (!net->ipv4.nat_bysource)
return -ENOMEM;
return 0;
Expand Down
Loading

0 comments on commit ea781f1

Please sign in to comment.