Skip to content

Commit

Permalink
udp6: don't make extra copies of iflow
Browse files Browse the repository at this point in the history 
udpv6_sendmsg() first initialises an on-stack 88B struct flowi6 and then
copies it into cork, which is expensive. Avoid the copy in corkless case
by initialising on-stack cork->fl directly.

The main part is a couple of lines under !corkreq check. The rest
converts fl6 variable to be a pointer.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
  • Loading branch information
Pavel Begunkov authored and Jakub Kicinski committed Jan 28, 2022
1 parent f37a4cc commit 5298953
Showing 1 changed file with 42 additions and 43 deletions.
85 changes: 42 additions & 43 deletions net/ipv6/udp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1294,7 +1294,8 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
struct ipv6_txoptions *opt = NULL;
struct ipv6_txoptions *opt_to_free = NULL;
struct ip6_flowlabel *flowlabel = NULL;
struct flowi6 fl6;
struct inet_cork_full cork;
struct flowi6 *fl6 = &cork.fl.u.ip6;
struct dst_entry *dst;
struct ipcm6_cookie ipc6;
int addr_len = msg->msg_namelen;
Expand Down Expand Up @@ -1384,19 +1385,19 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
}
ulen += sizeof(struct udphdr);

memset(&fl6, 0, sizeof(fl6));
memset(fl6, 0, sizeof(*fl6));

if (sin6) {
if (sin6->sin6_port == 0)
return -EINVAL;

fl6.fl6_dport = sin6->sin6_port;
fl6->fl6_dport = sin6->sin6_port;
daddr = &sin6->sin6_addr;

if (np->sndflow) {
fl6.flowlabel = sin6->sin6_flowinfo&IPV6_FLOWINFO_MASK;
if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
fl6->flowlabel = sin6->sin6_flowinfo&IPV6_FLOWINFO_MASK;
if (fl6->flowlabel & IPV6_FLOWLABEL_MASK) {
flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
if (IS_ERR(flowlabel))
return -EINVAL;
}
Expand All @@ -1413,24 +1414,24 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
if (addr_len >= sizeof(struct sockaddr_in6) &&
sin6->sin6_scope_id &&
__ipv6_addr_needs_scope_id(__ipv6_addr_type(daddr)))
fl6.flowi6_oif = sin6->sin6_scope_id;
fl6->flowi6_oif = sin6->sin6_scope_id;
} else {
if (sk->sk_state != TCP_ESTABLISHED)
return -EDESTADDRREQ;

fl6.fl6_dport = inet->inet_dport;
fl6->fl6_dport = inet->inet_dport;
daddr = &sk->sk_v6_daddr;
fl6.flowlabel = np->flow_label;
fl6->flowlabel = np->flow_label;
connected = true;
}

if (!fl6.flowi6_oif)
fl6.flowi6_oif = sk->sk_bound_dev_if;
if (!fl6->flowi6_oif)
fl6->flowi6_oif = sk->sk_bound_dev_if;

if (!fl6.flowi6_oif)
fl6.flowi6_oif = np->sticky_pktinfo.ipi6_ifindex;
if (!fl6->flowi6_oif)
fl6->flowi6_oif = np->sticky_pktinfo.ipi6_ifindex;

fl6.flowi6_uid = sk->sk_uid;
fl6->flowi6_uid = sk->sk_uid;

if (msg->msg_controllen) {
opt = &opt_space;
Expand All @@ -1440,14 +1441,14 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)

err = udp_cmsg_send(sk, msg, &ipc6.gso_size);
if (err > 0)
err = ip6_datagram_send_ctl(sock_net(sk), sk, msg, &fl6,
err = ip6_datagram_send_ctl(sock_net(sk), sk, msg, fl6,
&ipc6);
if (err < 0) {
fl6_sock_release(flowlabel);
return err;
}
if ((fl6.flowlabel&IPV6_FLOWLABEL_MASK) && !flowlabel) {
flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
if ((fl6->flowlabel&IPV6_FLOWLABEL_MASK) && !flowlabel) {
flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
if (IS_ERR(flowlabel))
return -EINVAL;
}
Expand All @@ -1464,16 +1465,17 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
opt = ipv6_fixup_options(&opt_space, opt);
ipc6.opt = opt;

fl6.flowi6_proto = sk->sk_protocol;
fl6.flowi6_mark = ipc6.sockc.mark;
fl6.daddr = *daddr;
if (ipv6_addr_any(&fl6.saddr) && !ipv6_addr_any(&np->saddr))
fl6.saddr = np->saddr;
fl6.fl6_sport = inet->inet_sport;
fl6->flowi6_proto = sk->sk_protocol;
fl6->flowi6_mark = ipc6.sockc.mark;
fl6->daddr = *daddr;
if (ipv6_addr_any(&fl6->saddr) && !ipv6_addr_any(&np->saddr))
fl6->saddr = np->saddr;
fl6->fl6_sport = inet->inet_sport;

if (cgroup_bpf_enabled(CGROUP_UDP6_SENDMSG) && !connected) {
err = BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk,
(struct sockaddr *)sin6, &fl6.saddr);
(struct sockaddr *)sin6,
&fl6->saddr);
if (err)
goto out_no_dst;
if (sin6) {
Expand All @@ -1489,59 +1491,56 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
err = -EINVAL;
goto out_no_dst;
}
fl6.fl6_dport = sin6->sin6_port;
fl6.daddr = sin6->sin6_addr;
fl6->fl6_dport = sin6->sin6_port;
fl6->daddr = sin6->sin6_addr;
}
}

if (ipv6_addr_any(&fl6.daddr))
fl6.daddr.s6_addr[15] = 0x1; /* :: means loopback (BSD'ism) */
if (ipv6_addr_any(&fl6->daddr))
fl6->daddr.s6_addr[15] = 0x1; /* :: means loopback (BSD'ism) */

final_p = fl6_update_dst(&fl6, opt, &final);
final_p = fl6_update_dst(fl6, opt, &final);
if (final_p)
connected = false;

if (!fl6.flowi6_oif && ipv6_addr_is_multicast(&fl6.daddr)) {
fl6.flowi6_oif = np->mcast_oif;
if (!fl6->flowi6_oif && ipv6_addr_is_multicast(&fl6->daddr)) {
fl6->flowi6_oif = np->mcast_oif;
connected = false;
} else if (!fl6.flowi6_oif)
fl6.flowi6_oif = np->ucast_oif;
} else if (!fl6->flowi6_oif)
fl6->flowi6_oif = np->ucast_oif;

security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6));

if (ipc6.tclass < 0)
ipc6.tclass = np->tclass;

fl6.flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6.flowlabel);
fl6->flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6->flowlabel);

dst = ip6_sk_dst_lookup_flow(sk, &fl6, final_p, connected);
dst = ip6_sk_dst_lookup_flow(sk, fl6, final_p, connected);
if (IS_ERR(dst)) {
err = PTR_ERR(dst);
dst = NULL;
goto out;
}

if (ipc6.hlimit < 0)
ipc6.hlimit = ip6_sk_dst_hoplimit(np, &fl6, dst);
ipc6.hlimit = ip6_sk_dst_hoplimit(np, fl6, dst);

if (msg->msg_flags&MSG_CONFIRM)
goto do_confirm;
back_from_confirm:

/* Lockless fast path for the non-corking case */
if (!corkreq) {
struct inet_cork_full cork;
struct sk_buff *skb;

cork.fl.u.ip6 = fl6;

skb = ip6_make_skb(sk, getfrag, msg, ulen,
sizeof(struct udphdr), &ipc6,
(struct rt6_info *)dst,
msg->msg_flags, &cork);
err = PTR_ERR(skb);
if (!IS_ERR_OR_NULL(skb))
err = udp_v6_send_skb(skb, &fl6, &cork.base);
err = udp_v6_send_skb(skb, fl6, &cork.base);
goto out;
}

Expand All @@ -1563,7 +1562,7 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
ipc6.dontfrag = np->dontfrag;
up->len += ulen;
err = ip6_append_data(sk, getfrag, msg, ulen, sizeof(struct udphdr),
&ipc6, &fl6, (struct rt6_info *)dst,
&ipc6, fl6, (struct rt6_info *)dst,
corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
if (err)
udp_v6_flush_pending_frames(sk);
Expand Down Expand Up @@ -1598,7 +1597,7 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)

do_confirm:
if (msg->msg_flags & MSG_PROBE)
dst_confirm_neigh(dst, &fl6.daddr);
dst_confirm_neigh(dst, &fl6->daddr);
if (!(msg->msg_flags&MSG_PROBE) || len)
goto back_from_confirm;
err = 0;
Expand Down

0 comments on commit 5298953

Please sign in to comment.