Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history
yaml
---
r: 328094
b: refs/heads/master
c: 5640f76
h: refs/heads/master
v: v3
  • Loading branch information
Eric Dumazet authored and David S. Miller committed Sep 24, 2012
1 parent e1412b6 commit 4078ea3
Show file tree
Hide file tree
Showing 14 changed files with 168 additions and 201 deletions.
2 changes: 1 addition & 1 deletion [refs]
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: b98b8babd6e3370fadb7c6eaacb00eb2f6344a6c
refs/heads/master: 5640f7685831e088fe6c2e1f863a6805962f8e81
3 changes: 3 additions & 0 deletions trunk/include/linux/sched.h
Original file line number Diff line number Diff line change
Expand Up @@ -1530,6 +1530,9 @@ struct task_struct {
* cache last used pipe for splice
*/
struct pipe_inode_info *splice_pipe;

struct page_frag task_frag;

#ifdef CONFIG_TASK_DELAY_ACCT
struct task_delay_info *delays;
#endif
Expand Down
4 changes: 1 addition & 3 deletions trunk/include/net/inet_sock.h
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,8 @@ struct inet_cork {
__be32 addr;
struct ip_options *opt;
unsigned int fragsize;
struct dst_entry *dst;
int length; /* Total length of all frames */
struct page *page;
u32 off;
struct dst_entry *dst;
u8 tx_flags;
};

Expand Down
27 changes: 15 additions & 12 deletions trunk/include/net/sock.h
Original file line number Diff line number Diff line change
Expand Up @@ -247,8 +247,7 @@ struct cg_proto;
* @sk_stamp: time stamp of last packet received
* @sk_socket: Identd and reporting IO signals
* @sk_user_data: RPC layer private data
* @sk_sndmsg_page: cached page for sendmsg
* @sk_sndmsg_off: cached offset for sendmsg
* @sk_frag: cached page frag
* @sk_peek_off: current peek_offset value
* @sk_send_head: front of stuff to transmit
* @sk_security: used by security modules
Expand Down Expand Up @@ -362,9 +361,8 @@ struct sock {
ktime_t sk_stamp;
struct socket *sk_socket;
void *sk_user_data;
struct page *sk_sndmsg_page;
struct page_frag sk_frag;
struct sk_buff *sk_send_head;
__u32 sk_sndmsg_off;
__s32 sk_peek_off;
int sk_write_pending;
#ifdef CONFIG_SECURITY
Expand Down Expand Up @@ -2034,18 +2032,23 @@ static inline void sk_stream_moderate_sndbuf(struct sock *sk)

struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp);

static inline struct page *sk_stream_alloc_page(struct sock *sk)
/**
* sk_page_frag - return an appropriate page_frag
* @sk: socket
*
* If socket allocation mode allows current thread to sleep, it means its
* safe to use the per task page_frag instead of the per socket one.
*/
static inline struct page_frag *sk_page_frag(struct sock *sk)
{
struct page *page = NULL;
if (sk->sk_allocation & __GFP_WAIT)
return &current->task_frag;

page = alloc_pages(sk->sk_allocation, 0);
if (!page) {
sk_enter_memory_pressure(sk);
sk_stream_moderate_sndbuf(sk);
}
return page;
return &sk->sk_frag;
}

extern bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);

/*
* Default write policy as shown to user space via poll/select/SIGIO
*/
Expand Down
3 changes: 3 additions & 0 deletions trunk/kernel/exit.c
Original file line number Diff line number Diff line change
Expand Up @@ -1046,6 +1046,9 @@ void do_exit(long code)
if (tsk->splice_pipe)
__free_pipe_info(tsk->splice_pipe);

if (tsk->task_frag.page)
put_page(tsk->task_frag.page);

validate_creds_for_do_exit(tsk);

preempt_disable();
Expand Down
1 change: 1 addition & 0 deletions trunk/kernel/fork.c
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
tsk->btrace_seq = 0;
#endif
tsk->splice_pipe = NULL;
tsk->task_frag.page = NULL;

account_kernel_stack(ti, 1);

Expand Down
37 changes: 9 additions & 28 deletions trunk/net/core/skbuff.c
Original file line number Diff line number Diff line change
Expand Up @@ -1655,38 +1655,19 @@ static struct page *linear_to_page(struct page *page, unsigned int *len,
unsigned int *offset,
struct sk_buff *skb, struct sock *sk)
{
struct page *p = sk->sk_sndmsg_page;
unsigned int off;
struct page_frag *pfrag = sk_page_frag(sk);

if (!p) {
new_page:
p = sk->sk_sndmsg_page = alloc_pages(sk->sk_allocation, 0);
if (!p)
return NULL;

off = sk->sk_sndmsg_off = 0;
/* hold one ref to this page until it's full */
} else {
unsigned int mlen;

/* If we are the only user of the page, we can reset offset */
if (page_count(p) == 1)
sk->sk_sndmsg_off = 0;
off = sk->sk_sndmsg_off;
mlen = PAGE_SIZE - off;
if (mlen < 64 && mlen < *len) {
put_page(p);
goto new_page;
}
if (!sk_page_frag_refill(sk, pfrag))
return NULL;

*len = min_t(unsigned int, *len, mlen);
}
*len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);

memcpy(page_address(p) + off, page_address(page) + *offset, *len);
sk->sk_sndmsg_off += *len;
*offset = off;
memcpy(page_address(pfrag->page) + pfrag->offset,
page_address(page) + *offset, *len);
*offset = pfrag->offset;
pfrag->offset += *len;

return p;
return pfrag->page;
}

static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
Expand Down
49 changes: 47 additions & 2 deletions trunk/net/core/sock.c
Original file line number Diff line number Diff line change
Expand Up @@ -1744,6 +1744,45 @@ struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
}
EXPORT_SYMBOL(sock_alloc_send_skb);

/* On 32bit arches, an skb frag is limited to 2^15 */
#define SKB_FRAG_PAGE_ORDER get_order(32768)

bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
{
int order;

if (pfrag->page) {
if (atomic_read(&pfrag->page->_count) == 1) {
pfrag->offset = 0;
return true;
}
if (pfrag->offset < pfrag->size)
return true;
put_page(pfrag->page);
}

/* We restrict high order allocations to users that can afford to wait */
order = (sk->sk_allocation & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;

do {
gfp_t gfp = sk->sk_allocation;

if (order)
gfp |= __GFP_COMP | __GFP_NOWARN;
pfrag->page = alloc_pages(gfp, order);
if (likely(pfrag->page)) {
pfrag->offset = 0;
pfrag->size = PAGE_SIZE << order;
return true;
}
} while (--order >= 0);

sk_enter_memory_pressure(sk);
sk_stream_moderate_sndbuf(sk);
return false;
}
EXPORT_SYMBOL(sk_page_frag_refill);

static void __lock_sock(struct sock *sk)
__releases(&sk->sk_lock.slock)
__acquires(&sk->sk_lock.slock)
Expand Down Expand Up @@ -2173,8 +2212,8 @@ void sock_init_data(struct socket *sock, struct sock *sk)
sk->sk_error_report = sock_def_error_report;
sk->sk_destruct = sock_def_destruct;

sk->sk_sndmsg_page = NULL;
sk->sk_sndmsg_off = 0;
sk->sk_frag.page = NULL;
sk->sk_frag.offset = 0;
sk->sk_peek_off = -1;

sk->sk_peer_pid = NULL;
Expand Down Expand Up @@ -2417,6 +2456,12 @@ void sk_common_release(struct sock *sk)
xfrm_sk_free_policy(sk);

sk_refcnt_debug_release(sk);

if (sk->sk_frag.page) {
put_page(sk->sk_frag.page);
sk->sk_frag.page = NULL;
}

sock_put(sk);
}
EXPORT_SYMBOL(sk_common_release);
Expand Down
70 changes: 28 additions & 42 deletions trunk/net/ipv4/ip_output.c
Original file line number Diff line number Diff line change
Expand Up @@ -793,6 +793,7 @@ static int __ip_append_data(struct sock *sk,
struct flowi4 *fl4,
struct sk_buff_head *queue,
struct inet_cork *cork,
struct page_frag *pfrag,
int getfrag(void *from, char *to, int offset,
int len, int odd, struct sk_buff *skb),
void *from, int length, int transhdrlen,
Expand Down Expand Up @@ -987,47 +988,30 @@ static int __ip_append_data(struct sock *sk,
}
} else {
int i = skb_shinfo(skb)->nr_frags;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
struct page *page = cork->page;
int off = cork->off;
unsigned int left;

if (page && (left = PAGE_SIZE - off) > 0) {
if (copy >= left)
copy = left;
if (page != skb_frag_page(frag)) {
if (i == MAX_SKB_FRAGS) {
err = -EMSGSIZE;
goto error;
}
skb_fill_page_desc(skb, i, page, off, 0);
skb_frag_ref(skb, i);
frag = &skb_shinfo(skb)->frags[i];
}
} else if (i < MAX_SKB_FRAGS) {
if (copy > PAGE_SIZE)
copy = PAGE_SIZE;
page = alloc_pages(sk->sk_allocation, 0);
if (page == NULL) {
err = -ENOMEM;
goto error;
}
cork->page = page;
cork->off = 0;

skb_fill_page_desc(skb, i, page, 0, 0);
frag = &skb_shinfo(skb)->frags[i];
} else {
err = -EMSGSIZE;
goto error;
}
if (getfrag(from, skb_frag_address(frag)+skb_frag_size(frag),
offset, copy, skb->len, skb) < 0) {
err = -EFAULT;
err = -ENOMEM;
if (!sk_page_frag_refill(sk, pfrag))
goto error;

if (!skb_can_coalesce(skb, i, pfrag->page,
pfrag->offset)) {
err = -EMSGSIZE;
if (i == MAX_SKB_FRAGS)
goto error;

__skb_fill_page_desc(skb, i, pfrag->page,
pfrag->offset, 0);
skb_shinfo(skb)->nr_frags = ++i;
get_page(pfrag->page);
}
cork->off += copy;
skb_frag_size_add(frag, copy);
copy = min_t(int, copy, pfrag->size - pfrag->offset);
if (getfrag(from,
page_address(pfrag->page) + pfrag->offset,
offset, copy, skb->len, skb) < 0)
goto error_efault;

pfrag->offset += copy;
skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
skb->len += copy;
skb->data_len += copy;
skb->truesize += copy;
Expand All @@ -1039,6 +1023,8 @@ static int __ip_append_data(struct sock *sk,

return 0;

error_efault:
err = -EFAULT;
error:
cork->length -= length;
IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
Expand Down Expand Up @@ -1079,8 +1065,6 @@ static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
cork->dst = &rt->dst;
cork->length = 0;
cork->tx_flags = ipc->tx_flags;
cork->page = NULL;
cork->off = 0;

return 0;
}
Expand Down Expand Up @@ -1117,7 +1101,8 @@ int ip_append_data(struct sock *sk, struct flowi4 *fl4,
transhdrlen = 0;
}

return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base, getfrag,
return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base,
sk_page_frag(sk), getfrag,
from, length, transhdrlen, flags);
}

Expand Down Expand Up @@ -1439,7 +1424,8 @@ struct sk_buff *ip_make_skb(struct sock *sk,
if (err)
return ERR_PTR(err);

err = __ip_append_data(sk, fl4, &queue, &cork, getfrag,
err = __ip_append_data(sk, fl4, &queue, &cork,
&current->task_frag, getfrag,
from, length, transhdrlen, flags);
if (err) {
__ip_flush_pending_frames(sk, &queue, &cork);
Expand Down
19 changes: 12 additions & 7 deletions trunk/net/ipv4/raw.c
Original file line number Diff line number Diff line change
Expand Up @@ -131,18 +131,23 @@ static struct sock *__raw_v4_lookup(struct net *net, struct sock *sk,
* 0 - deliver
* 1 - block
*/
static __inline__ int icmp_filter(struct sock *sk, struct sk_buff *skb)
static int icmp_filter(const struct sock *sk, const struct sk_buff *skb)
{
int type;

if (!pskb_may_pull(skb, sizeof(struct icmphdr)))
struct icmphdr _hdr;
const struct icmphdr *hdr;

pr_err("icmp_filter skb_transport_offset %d data-head %ld len %d/%d\n",
skb_transport_offset(skb), skb->data - skb->head, skb->len, skb->data_len);
hdr = skb_header_pointer(skb, skb_transport_offset(skb),
sizeof(_hdr), &_hdr);
pr_err("head %p data %p hdr %p type %d\n", skb->head, skb->data, hdr, hdr ? hdr->type : -1);
if (!hdr)
return 1;

type = icmp_hdr(skb)->type;
if (type < 32) {
if (hdr->type < 32) {
__u32 data = raw_sk(sk)->filter.data;

return ((1 << type) & data) != 0;
return ((1U << hdr->type) & data) != 0;
}

/* Do not block unknown ICMP types */
Expand Down
Loading

0 comments on commit 4078ea3

Please sign in to comment.