Skip to content

Commit

Permalink
Merge branch 'tcp-lockless-listener'
Browse files Browse the repository at this point in the history 
Eric Dumazet says:

====================
tcp/dccp: lockless listener

TCP listener refactoring : this is becoming interesting !

This patch series takes the steps to use normal TCP/DCCP ehash
table to store SYN_RECV requests, instead of the private per-listener
hash table we had until now.

SYNACK skb are now attached to their syn_recv request socket,
so that we no longer heavily modify listener sk_wmem_alloc.

listener lock is no longer held in fast path, including
SYNCOOKIE mode.

During my tests, my server was able to process 3,500,000
SYN packets per second on one listener and still had available
cpu cycles.

That is about 2 to 3 order of magnitude what we had with older kernels.

This effort started two years ago and I am pleased to reach expectations.

We'll probably extend SO_REUSEPORT to add proper cpu/numa affinities,
so that heavy duty TCP servers can get proper siloing thanks to multi-queues
NIC.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
David S. Miller committed Oct 3, 2015
2 parents f6d3125 + e994b2f commit c3fc7ac
Show file tree
Hide file tree
Showing 20 changed files with 307 additions and 746 deletions.
9 changes: 0 additions & 9 deletions include/net/inet6_connection_sock.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,6 @@ int inet6_csk_bind_conflict(const struct sock *sk,
struct dst_entry *inet6_csk_route_req(const struct sock *sk, struct flowi6 *fl6,
const struct request_sock *req, u8 proto);

struct request_sock *inet6_csk_search_req(struct sock *sk,
const __be16 rport,
const struct in6_addr *raddr,
const struct in6_addr *laddr,
const int iif);

void inet6_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req,
const unsigned long timeout);

void inet6_csk_addr2sockaddr(struct sock *sk, struct sockaddr *uaddr);

int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl);
Expand Down
9 changes: 2 additions & 7 deletions include/net/inet_connection_sock.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,10 +258,6 @@ inet_csk_rto_backoff(const struct inet_connection_sock *icsk,

struct sock *inet_csk_accept(struct sock *sk, int flags, int *err);

struct request_sock *inet_csk_search_req(struct sock *sk,
const __be16 rport,
const __be32 raddr,
const __be32 laddr);
int inet_csk_bind_conflict(const struct sock *sk,
const struct inet_bind_bucket *tb, bool relax);
int inet_csk_get_port(struct sock *sk, unsigned short snum);
Expand All @@ -282,8 +278,7 @@ static inline void inet_csk_reqsk_queue_add(struct sock *sk,
void inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req,
unsigned long timeout);

static inline void inet_csk_reqsk_queue_added(struct sock *sk,
const unsigned long timeout)
static inline void inet_csk_reqsk_queue_added(struct sock *sk)
{
reqsk_queue_added(&inet_csk(sk)->icsk_accept_queue);
}
Expand All @@ -300,7 +295,7 @@ static inline int inet_csk_reqsk_queue_young(const struct sock *sk)

static inline int inet_csk_reqsk_queue_is_full(const struct sock *sk)
{
return reqsk_queue_is_full(&inet_csk(sk)->icsk_accept_queue);
return inet_csk_reqsk_queue_len(sk) >= sk->sk_max_ack_backlog;
}

void inet_csk_reqsk_queue_drop(struct sock *sk, struct request_sock *req);
Expand Down
1 change: 1 addition & 0 deletions include/net/inet_hashtables.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ void inet_put_port(struct sock *sk);

void inet_hashinfo_init(struct inet_hashinfo *h);

int inet_ehash_insert(struct sock *sk, struct sock *osk);
void __inet_hash_nolisten(struct sock *sk, struct sock *osk);
void __inet_hash(struct sock *sk, struct sock *osk);
void inet_hash(struct sock *sk);
Expand Down
132 changes: 41 additions & 91 deletions include/net/request_sock.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,16 @@ struct request_sock {
u32 peer_secid;
};

static inline struct request_sock *inet_reqsk(struct sock *sk)
{
return (struct request_sock *)sk;
}

static inline struct sock *req_to_sk(struct request_sock *req)
{
return (struct sock *)req;
}

static inline struct request_sock *
reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener)
{
Expand All @@ -78,6 +88,8 @@ reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener)
req->rsk_ops = ops;
sock_hold(sk_listener);
req->rsk_listener = sk_listener;
req_to_sk(req)->sk_prot = sk_listener->sk_prot;
sk_node_init(&req_to_sk(req)->sk_node);
req->saved_syn = NULL;
/* Following is temporary. It is coupled with debugging
* helpers in reqsk_put() & reqsk_free()
Expand All @@ -87,16 +99,6 @@ reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener)
return req;
}

static inline struct request_sock *inet_reqsk(struct sock *sk)
{
return (struct request_sock *)sk;
}

static inline struct sock *req_to_sk(struct request_sock *req)
{
return (struct sock *)req;
}

static inline void reqsk_free(struct request_sock *req)
{
/* temporary debugging */
Expand All @@ -117,25 +119,6 @@ static inline void reqsk_put(struct request_sock *req)

extern int sysctl_max_syn_backlog;

/** struct listen_sock - listen state
*
* @max_qlen_log - log_2 of maximal queued SYNs/REQUESTs
*/
struct listen_sock {
int qlen_inc; /* protected by listener lock */
int young_inc;/* protected by listener lock */

/* following fields can be updated by timer */
atomic_t qlen_dec; /* qlen = qlen_inc - qlen_dec */
atomic_t young_dec;

u32 max_qlen_log ____cacheline_aligned_in_smp;
u32 synflood_warned;
u32 hash_rnd;
u32 nr_table_entries;
struct request_sock *syn_table[0];
};

/*
* For a TCP Fast Open listener -
* lock - protects the access to all the reqsk, which is co-owned by
Expand Down Expand Up @@ -169,43 +152,29 @@ struct fastopen_queue {
* @rskq_accept_head - FIFO head of established children
* @rskq_accept_tail - FIFO tail of established children
* @rskq_defer_accept - User waits for some data after accept()
* @syn_wait_lock - serializer
*
* %syn_wait_lock is necessary only to avoid proc interface having to grab the main
* lock sock while browsing the listening hash (otherwise it's deadlock prone).
*
*/
struct request_sock_queue {
spinlock_t rskq_lock;
u8 rskq_defer_accept;

u32 synflood_warned;
atomic_t qlen;
atomic_t young;

struct request_sock *rskq_accept_head;
struct request_sock *rskq_accept_tail;
u8 rskq_defer_accept;
struct listen_sock *listen_opt;
struct fastopen_queue fastopenq; /* Check max_qlen != 0 to determine
* if TFO is enabled.
*/

/* temporary alignment, our goal is to get rid of this lock */
spinlock_t syn_wait_lock ____cacheline_aligned_in_smp;
};

int reqsk_queue_alloc(struct request_sock_queue *queue,
unsigned int nr_table_entries);
void reqsk_queue_alloc(struct request_sock_queue *queue);

void __reqsk_queue_destroy(struct request_sock_queue *queue);
void reqsk_queue_destroy(struct request_sock_queue *queue);
void reqsk_fastopen_remove(struct sock *sk, struct request_sock *req,
bool reset);

static inline struct request_sock *
reqsk_queue_yank_acceptq(struct request_sock_queue *queue)
{
struct request_sock *req = queue->rskq_accept_head;

queue->rskq_accept_head = NULL;
return req;
}

static inline int reqsk_queue_empty(struct request_sock_queue *queue)
static inline bool reqsk_queue_empty(const struct request_sock_queue *queue)
{
return queue->rskq_accept_head == NULL;
}
Expand All @@ -215,6 +184,7 @@ static inline void reqsk_queue_add(struct request_sock_queue *queue,
struct sock *parent,
struct sock *child)
{
spin_lock(&queue->rskq_lock);
req->sk = child;
sk_acceptq_added(parent);

Expand All @@ -225,68 +195,48 @@ static inline void reqsk_queue_add(struct request_sock_queue *queue,

queue->rskq_accept_tail = req;
req->dl_next = NULL;
spin_unlock(&queue->rskq_lock);
}

static inline struct request_sock *reqsk_queue_remove(struct request_sock_queue *queue)
static inline struct request_sock *reqsk_queue_remove(struct request_sock_queue *queue,
struct sock *parent)
{
struct request_sock *req = queue->rskq_accept_head;

WARN_ON(req == NULL);

queue->rskq_accept_head = req->dl_next;
if (queue->rskq_accept_head == NULL)
queue->rskq_accept_tail = NULL;
struct request_sock *req;

spin_lock_bh(&queue->rskq_lock);
req = queue->rskq_accept_head;
if (req) {
sk_acceptq_removed(parent);
queue->rskq_accept_head = req->dl_next;
if (queue->rskq_accept_head == NULL)
queue->rskq_accept_tail = NULL;
}
spin_unlock_bh(&queue->rskq_lock);
return req;
}

static inline void reqsk_queue_removed(struct request_sock_queue *queue,
const struct request_sock *req)
{
struct listen_sock *lopt = queue->listen_opt;

if (req->num_timeout == 0)
atomic_inc(&lopt->young_dec);
atomic_inc(&lopt->qlen_dec);
atomic_dec(&queue->young);
atomic_dec(&queue->qlen);
}

static inline void reqsk_queue_added(struct request_sock_queue *queue)
{
struct listen_sock *lopt = queue->listen_opt;

lopt->young_inc++;
lopt->qlen_inc++;
}

static inline int listen_sock_qlen(const struct listen_sock *lopt)
{
return lopt->qlen_inc - atomic_read(&lopt->qlen_dec);
}

static inline int listen_sock_young(const struct listen_sock *lopt)
{
return lopt->young_inc - atomic_read(&lopt->young_dec);
atomic_inc(&queue->young);
atomic_inc(&queue->qlen);
}

static inline int reqsk_queue_len(const struct request_sock_queue *queue)
{
const struct listen_sock *lopt = queue->listen_opt;

return lopt ? listen_sock_qlen(lopt) : 0;
return atomic_read(&queue->qlen);
}

static inline int reqsk_queue_len_young(const struct request_sock_queue *queue)
{
return listen_sock_young(queue->listen_opt);
return atomic_read(&queue->young);
}

static inline int reqsk_queue_is_full(const struct request_sock_queue *queue)
{
return reqsk_queue_len(queue) >> queue->listen_opt->max_qlen_log;
}

void reqsk_queue_hash_req(struct request_sock_queue *queue,
u32 hash, struct request_sock *req,
unsigned long timeout);

#endif /* _REQUEST_SOCK_H */
10 changes: 4 additions & 6 deletions include/net/tcp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,8 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
int tcp_connect(struct sock *sk);
struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
struct request_sock *req,
struct tcp_fastopen_cookie *foc);
struct tcp_fastopen_cookie *foc,
bool attach_req);
int tcp_disconnect(struct sock *sk, int flags);

void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
Expand Down Expand Up @@ -1618,7 +1619,6 @@ static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
/* /proc */
enum tcp_seq_states {
TCP_SEQ_STATE_LISTENING,
TCP_SEQ_STATE_OPENREQ,
TCP_SEQ_STATE_ESTABLISHED,
};

Expand All @@ -1637,7 +1637,6 @@ struct tcp_iter_state {
enum tcp_seq_states state;
struct sock *syn_wait_sk;
int bucket, offset, sbucket, num;
kuid_t uid;
loff_t last_pos;
};

Expand Down Expand Up @@ -1717,9 +1716,8 @@ struct tcp_request_sock_ops {
__u32 (*init_seq)(const struct sk_buff *skb);
int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
struct flowi *fl, struct request_sock *req,
u16 queue_mapping, struct tcp_fastopen_cookie *foc);
void (*queue_hash_add)(struct sock *sk, struct request_sock *req,
const unsigned long timeout);
u16 queue_mapping, struct tcp_fastopen_cookie *foc,
bool attach_req);
};

#ifdef CONFIG_SYN_COOKIES
Expand Down
Loading

0 comments on commit c3fc7ac

Please sign in to comment.