cregit-Linux how code gets into the kernel

Release 4.14 net/ipv6/udp.c

Directory: net/ipv6
/*
 *      UDP over IPv6
 *      Linux INET6 implementation
 *
 *      Authors:
 *      Pedro Roque             <roque@di.fc.ul.pt>
 *
 *      Based on linux/ipv4/udp.c
 *
 *      Fixes:
 *      Hideaki YOSHIFUJI       :       sin6_scope_id support
 *      YOSHIFUJI Hideaki @USAGI and:   Support IPV6_V6ONLY socket option, which
 *      Alexey Kuznetsov                allow both IPv4 and IPv6 sockets to bind
 *                                      a single port at the same time.
 *      Kazunori MIYAZAWA @USAGI:       change process style to use ip6_append_data
 *      YOSHIFUJI Hideaki @USAGI:       convert /proc/net/udp6 to seq_file.
 *
 *      This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 */

#include <linux/errno.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/in6.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/ipv6.h>
#include <linux/icmpv6.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/uaccess.h>

#include <net/addrconf.h>
#include <net/ndisc.h>
#include <net/protocol.h>
#include <net/transp_v6.h>
#include <net/ip6_route.h>
#include <net/raw.h>
#include <net/tcp_states.h>
#include <net/ip6_checksum.h>
#include <net/xfrm.h>
#include <net/inet_hashtables.h>
#include <net/inet6_hashtables.h>
#include <net/busy_poll.h>
#include <net/sock_reuseport.h>

#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <trace/events/skb.h>
#include "udp_impl.h"


static bool udp6_lib_exact_dif_match(struct net *net, struct sk_buff *skb) { #if defined(CONFIG_NET_L3_MASTER_DEV) if (!net->ipv4.sysctl_udp_l3mdev_accept && skb && ipv6_l3mdev_skb(IP6CB(skb)->flags)) return true; #endif return false; }

Contributors

PersonTokensPropCommitsCommitProp
Robert Shearman51100.00%1100.00%
Total51100.00%1100.00%


static u32 udp6_ehashfn(const struct net *net, const struct in6_addr *laddr, const u16 lport, const struct in6_addr *faddr, const __be16 fport) { static u32 udp6_ehash_secret __read_mostly; static u32 udp_ipv6_hash_secret __read_mostly; u32 lhash, fhash; net_get_random_once(&udp6_ehash_secret, sizeof(udp6_ehash_secret)); net_get_random_once(&udp_ipv6_hash_secret, sizeof(udp_ipv6_hash_secret)); lhash = (__force u32)laddr->s6_addr32[3]; fhash = __ipv6_addr_jhash(faddr, udp_ipv6_hash_secret); return __inet6_ehashfn(lhash, lport, fhash, fport, udp_ipv6_hash_secret + net_hash_mix(net)); }

Contributors

PersonTokensPropCommitsCommitProp
Hannes Frederic Sowa10898.18%266.67%
Eric Dumazet21.82%133.33%
Total110100.00%3100.00%


static u32 udp6_portaddr_hash(const struct net *net, const struct in6_addr *addr6, unsigned int port) { unsigned int hash, mix = net_hash_mix(net); if (ipv6_addr_any(addr6)) hash = jhash_1word(0, mix); else if (ipv6_addr_v4mapped(addr6)) hash = jhash_1word((__force u32)addr6->s6_addr32[3], mix); else hash = jhash2((__force u32 *)addr6->s6_addr32, 4, mix); return hash ^ port; }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet9898.99%375.00%
Brian Haley11.01%125.00%
Total99100.00%4100.00%


int udp_v6_get_port(struct sock *sk, unsigned short snum) { unsigned int hash2_nulladdr = udp6_portaddr_hash(sock_net(sk), &in6addr_any, snum); unsigned int hash2_partial = udp6_portaddr_hash(sock_net(sk), &sk->sk_v6_rcv_saddr, 0); /* precompute partial secondary hash */ udp_sk(sk)->udp_portaddr_hash = hash2_partial; return udp_lib_get_port(sk, snum, hash2_nulladdr); }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet4767.14%325.00%
Linus Torvalds (pre-git)1724.29%541.67%
Arnaldo Carvalho de Melo45.71%216.67%
Pavel Emelyanov11.43%18.33%
Gerrit Renker11.43%18.33%
Total70100.00%12100.00%


static void udp_v6_rehash(struct sock *sk) { u16 new_hash = udp6_portaddr_hash(sock_net(sk), &sk->sk_v6_rcv_saddr, inet_sk(sk)->inet_num); udp_lib_rehash(sk, new_hash); }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet41100.00%2100.00%
Total41100.00%2100.00%


static int compute_score(struct sock *sk, struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, unsigned short hnum, int dif, int sdif, bool exact_dif) { int score; struct inet_sock *inet; if (!net_eq(sock_net(sk), net) || udp_sk(sk)->udp_port_hash != hnum || sk->sk_family != PF_INET6) return -1; score = 0; inet = inet_sk(sk); if (inet->inet_dport) { if (inet->inet_dport != sport) return -1; score++; } if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) { if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return -1; score++; } if (!ipv6_addr_any(&sk->sk_v6_daddr)) { if (!ipv6_addr_equal(&sk->sk_v6_daddr, saddr)) return -1; score++; } if (sk->sk_bound_dev_if || exact_dif) { bool dev_match = (sk->sk_bound_dev_if == dif || sk->sk_bound_dev_if == sdif); if (exact_dif && !dev_match) return -1; if (sk->sk_bound_dev_if && dev_match) score++; } if (sk->sk_incoming_cpu == raw_smp_processor_id()) score++; return score; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)10642.74%418.18%
Eric Dumazet5823.39%627.27%
David Ahern3012.10%14.55%
Joe Perches197.66%14.55%
Hideaki Yoshifuji / 吉藤英明114.44%313.64%
Pavel Emelyanov83.23%14.55%
Robert Shearman52.02%14.55%
Arnaldo Carvalho de Melo31.21%14.55%
Xuemin Su31.21%14.55%
Balazs Scheidler20.81%14.55%
David S. Miller20.81%14.55%
Gerrit Renker10.40%14.55%
Total248100.00%22100.00%

/* called with rcu_read_lock() */
static struct sock *udp6_lib_lookup2(struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, unsigned int hnum, int dif, int sdif, bool exact_dif, struct udp_hslot *hslot2, struct sk_buff *skb) { struct sock *sk, *result; int score, badness, matches = 0, reuseport = 0; u32 hash = 0; result = NULL; badness = -1; udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { score = compute_score(sk, net, saddr, sport, daddr, hnum, dif, sdif, exact_dif); if (score > badness) { reuseport = sk->sk_reuseport; if (reuseport) { hash = udp6_ehashfn(net, daddr, hnum, saddr, sport); result = reuseport_select_sock(sk, hash, skb, sizeof(struct udphdr)); if (result) return result; matches = 1; } result = sk; badness = score; } else if (score == badness && reuseport) { matches++; if (reciprocal_scale(hash, matches) == 0) result = sk; hash = next_pseudo_random32(hash); } } return result; }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet11249.34%325.00%
Tom Herbert7332.16%18.33%
Craig Gallek2611.45%325.00%
Robert Shearman52.20%18.33%
David Ahern52.20%18.33%
Daniel Borkmann41.76%18.33%
Hannes Frederic Sowa10.44%18.33%
Xuemin Su10.44%18.33%
Total227100.00%12100.00%

/* rcu_read_lock() must be held */
struct sock *__udp6_lib_lookup(struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, __be16 dport, int dif, int sdif, struct udp_table *udptable, struct sk_buff *skb) { struct sock *sk, *result; unsigned short hnum = ntohs(dport); unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask); struct udp_hslot *hslot2, *hslot = &udptable->hash[slot]; bool exact_dif = udp6_lib_exact_dif_match(net, skb); int score, badness, matches = 0, reuseport = 0; u32 hash = 0; if (hslot->count > 10) { hash2 = udp6_portaddr_hash(net, daddr, hnum); slot2 = hash2 & udptable->mask; hslot2 = &udptable->hash2[slot2]; if (hslot->count < hslot2->count) goto begin; result = udp6_lib_lookup2(net, saddr, sport, daddr, hnum, dif, sdif, exact_dif, hslot2, skb); if (!result) { unsigned int old_slot2 = slot2; hash2 = udp6_portaddr_hash(net, &in6addr_any, hnum); slot2 = hash2 & udptable->mask; /* avoid searching the same slot again. */ if (unlikely(slot2 == old_slot2)) return result; hslot2 = &udptable->hash2[slot2]; if (hslot->count < hslot2->count) goto begin; result = udp6_lib_lookup2(net, saddr, sport, daddr, hnum, dif, sdif, exact_dif, hslot2, skb); } return result; } begin: result = NULL; badness = -1; sk_for_each_rcu(sk, &hslot->head) { score = compute_score(sk, net, saddr, sport, daddr, hnum, dif, sdif, exact_dif); if (score > badness) { reuseport = sk->sk_reuseport; if (reuseport) { hash = udp6_ehashfn(net, daddr, hnum, saddr, sport); result = reuseport_select_sock(sk, hash, skb, sizeof(struct udphdr)); if (result) return result; matches = 1; } result = sk; badness = score; } else if (score == badness && reuseport) { matches++; if (reciprocal_scale(hash, matches) == 0) result = sk; hash = next_pseudo_random32(hash); } } return result; }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet27961.73%633.33%
Tom Herbert7215.93%15.56%
Craig Gallek296.42%316.67%
Xuemin Su214.65%15.56%
Linus Torvalds (pre-git)183.98%15.56%
Robert Shearman163.54%15.56%
David Ahern91.99%15.56%
Daniel Borkmann40.88%15.56%
Balazs Scheidler20.44%15.56%
Jorge Boncompte10.22%15.56%
Hannes Frederic Sowa10.22%15.56%
Total452100.00%18100.00%

EXPORT_SYMBOL_GPL(__udp6_lib_lookup);
static struct sock *__udp6_lib_lookup_skb(struct sk_buff *skb, __be16 sport, __be16 dport, struct udp_table *udptable) { const struct ipv6hdr *iph = ipv6_hdr(skb); return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport, &iph->daddr, dport, inet6_iif(skb), inet6_sdif(skb), udptable, skb); }

Contributors

PersonTokensPropCommitsCommitProp
KOVACS Krisztian6486.49%233.33%
David Ahern56.76%116.67%
Eric Dumazet34.05%233.33%
Craig Gallek22.70%116.67%
Total74100.00%6100.00%


struct sock *udp6_lib_lookup_skb(struct sk_buff *skb, __be16 sport, __be16 dport) { const struct ipv6hdr *iph = ipv6_hdr(skb); return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport, &iph->daddr, dport, inet6_iif(skb), inet6_sdif(skb), &udp_table, skb); }

Contributors

PersonTokensPropCommitsCommitProp
Tom Herbert6289.86%133.33%
David Ahern57.25%133.33%
Alexander Duyck22.90%133.33%
Total69100.00%3100.00%

EXPORT_SYMBOL_GPL(udp6_lib_lookup_skb); /* Must be called under rcu_read_lock(). * Does increment socket refcount. */ #if IS_ENABLED(CONFIG_NETFILTER_XT_MATCH_SOCKET) || \ IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TPROXY) || \ IS_ENABLED(CONFIG_NF_SOCKET_IPV6)
struct sock *udp6_lib_lookup(struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, __be16 dport, int dif) { struct sock *sk; sk = __udp6_lib_lookup(net, saddr, sport, daddr, dport, dif, 0, &udp_table, NULL); if (sk && !refcount_inc_not_zero(&sk->sk_refcnt)) sk = NULL; return sk; }

Contributors

PersonTokensPropCommitsCommitProp
Balazs Scheidler5060.98%120.00%
Eric Dumazet2732.93%120.00%
Craig Gallek22.44%120.00%
David Ahern22.44%120.00%
Elena Reshetova11.22%120.00%
Total82100.00%5100.00%

EXPORT_SYMBOL_GPL(udp6_lib_lookup); #endif /* do not use the scratch area len for jumbogram: their length execeeds the * scratch area space; note that the IP6CB flags is still in the first * cacheline, so checking for jumbograms is cheap */
static int udp6_skb_len(struct sk_buff *skb) { return unlikely(inet6_is_jumbogram(skb)) ? skb->len : udp_skb_len(skb); }

Contributors

PersonTokensPropCommitsCommitProp
Paolo Abeni29100.00%1100.00%
Total29100.00%1100.00%

/* * This should be easy, if there is something there we * return it, otherwise we block. */
int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock, int flags, int *addr_len) { struct ipv6_pinfo *np = inet6_sk(sk); struct inet_sock *inet = inet_sk(sk); struct sk_buff *skb; unsigned int ulen, copied; int peeked, peeking, off; int err; int is_udplite = IS_UDPLITE(sk); bool checksum_valid = false; int is_udp4; if (flags & MSG_ERRQUEUE) return ipv6_recv_error(sk, msg, len, addr_len); if (np->rxpmtu && np->rxopt.bits.rxpmtu) return ipv6_recv_rxpmtu(sk, msg, len, addr_len); try_again: peeking = flags & MSG_PEEK; off = sk_peek_offset(sk, flags); skb = __skb_recv_udp(sk, flags, noblock, &peeked, &off, &err); if (!skb) return err; ulen = udp6_skb_len(skb); copied = len; if (copied > ulen - off) copied = ulen - off; else if (copied < ulen) msg->msg_flags |= MSG_TRUNC; is_udp4 = (skb->protocol == htons(ETH_P_IP)); /* * If checksum is needed at all, try to do it while copying the * data. If the data is truncated, or if we only want a partial * coverage checksum (UDP-Lite), do it before the copy. */ if (copied < ulen || peeking || (is_udplite && UDP_SKB_CB(skb)->partial_cov)) { checksum_valid = udp_skb_csum_unnecessary(skb) || !__udp_lib_checksum_complete(skb); if (!checksum_valid) goto csum_copy_err; } if (checksum_valid || udp_skb_csum_unnecessary(skb)) { if (udp_skb_is_linear(skb)) err = copy_linear_skb(skb, copied, off, &msg->msg_iter); else err = skb_copy_datagram_msg(skb, off, msg, copied); } else { err = skb_copy_and_csum_datagram_msg(skb, off, msg); if (err == -EINVAL) goto csum_copy_err; } if (unlikely(err)) { if (!peeked) { atomic_inc(&sk->sk_drops); if (is_udp4) UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); else UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); } kfree_skb(skb); return err; } if (!peeked) { if (is_udp4) UDP_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS, is_udplite); else UDP6_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS, is_udplite); } sock_recv_ts_and_drops(msg, sk, skb); /* Copy the address. */ if (msg->msg_name) { DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name); sin6->sin6_family = AF_INET6; sin6->sin6_port = udp_hdr(skb)->source; sin6->sin6_flowinfo = 0; if (is_udp4) { ipv6_addr_set_v4mapped(ip_hdr(skb)->saddr, &sin6->sin6_addr); sin6->sin6_scope_id = 0; } else { sin6->sin6_addr = ipv6_hdr(skb)->saddr; sin6->sin6_scope_id = ipv6_iface_scope_id(&sin6->sin6_addr, inet6_iif(skb)); } *addr_len = sizeof(*sin6); } if (np->rxopt.all) ip6_datagram_recv_common_ctl(sk, msg, skb); if (is_udp4) { if (inet->cmsg_flags) ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off); } else { if (np->rxopt.all) ip6_datagram_recv_specific_ctl(sk, msg, skb); } err = copied; if (flags & MSG_TRUNC) err = ulen; skb_consume_udp(sk, skb, peeking ? -err : err); return err; csum_copy_err: if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags, udp_skb_destructor)) { if (is_udp4) { UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); } else { UDP6_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); } } kfree_skb(skb); /* starting over for a new packet, but check if we need to yield */ cond_resched(); msg->msg_flags &= ~MSG_TRUNC; goto try_again; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)21328.14%1015.87%
Eric Dumazet11715.46%914.29%
Wei Yongjun547.13%23.17%
Paolo Abeni537.00%69.52%
Hideaki Yoshifuji / 吉藤英明496.47%23.17%
Herbert Xu476.21%57.94%
Hannes Frederic Sowa466.08%46.35%
samanthakumar364.76%11.59%
Brian Haley303.96%23.17%
David S. Miller293.83%46.35%
Pavel Emelyanov151.98%23.17%
Gerrit Renker111.45%11.59%
Arnaldo Carvalho de Melo91.19%34.76%
Samuel Thibault91.19%11.59%
Xufeng Zhang70.92%11.59%
Wang Chen60.79%11.59%
Steffen Hurrle60.79%11.59%
Linus Torvalds50.66%11.59%
Matthew Dawson40.53%11.59%
Hideo Aoki40.53%11.59%
Stephen Hemminger30.40%11.59%
Duan Jiong10.13%11.59%
Alexey Dobriyan10.13%11.59%
Neil Horman10.13%11.59%
Al Viro10.13%11.59%
Total757100.00%63100.00%


void __udp6_lib_err(struct sk_buff *skb, struct inet6_skb_parm *opt, u8 type, u8 code, int offset, __be32 info, struct udp_table *udptable) { struct ipv6_pinfo *np; const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data; const struct in6_addr *saddr = &hdr->saddr; const struct in6_addr *daddr = &hdr->daddr; struct udphdr *uh = (struct udphdr *)(skb->data+offset); struct sock *sk; int harderr; int err; struct net *net = dev_net(skb->dev); sk = __udp6_lib_lookup(net, daddr, uh->dest, saddr, uh->source, inet6_iif(skb), 0, udptable, skb); if (!sk) { __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev), ICMP6_MIB_INERRORS); return; } harderr = icmpv6_err_convert(type, code, &err); np = inet6_sk(sk); if (type == ICMPV6_PKT_TOOBIG) { if (!ip6_sk_accept_pmtu(sk)) goto out; ip6_sk_update_pmtu(skb, sk, info); if (np->pmtudisc != IPV6_PMTUDISC_DONT) harderr = 1; } if (type == NDISC_REDIRECT) { ip6_sk_redirect(skb, sk); goto out; } if (!np->recverr) { if (!harderr || sk->sk_state != TCP_ESTABLISHED) goto out; } else { ipv6_icmp_error(sk, skb, err, uh->dest, ntohl(info), (u8 *)(uh+1)); } sk->sk_err = err; sk->sk_error_report(sk); out: return; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)12640.00%724.14%
Wei Wang4213.33%13.45%
David S. Miller3410.79%310.34%
Linus Torvalds3310.48%13.45%
Duan Jiong319.84%26.90%
Hannes Frederic Sowa134.13%13.45%
Eric Dumazet82.54%413.79%
Hideaki Yoshifuji / 吉藤英明72.22%26.90%
Gerrit Renker61.90%13.45%
Pavel Emelyanov41.27%13.45%
Arnaldo Carvalho de Melo30.95%13.45%
Craig Gallek20.63%13.45%
Brian Haley20.63%13.45%
David Ahern20.63%13.45%
Ian Morris10.32%13.45%
Al Viro10.32%13.45%
Total315100.00%29100.00%


static int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) { int rc; if (!ipv6_addr_any(&sk->sk_v6_daddr)) { sock_rps_save_rxhash(sk, skb); sk_mark_napi_id(sk, skb); sk_incoming_cpu_update(sk); } else { sk_mark_napi_id_once(sk, skb); } rc = __udp_enqueue_schedule_skb(sk, skb); if (rc < 0) { int is_udplite = IS_UDPLITE(sk); /* Note that an ENOMEM error is charged twice */ if (rc == -ENOMEM) UDP6_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS, is_udplite); UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); kfree_skb(skb); return -1; } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin LaHaise10177.69%112.50%
Eric Dumazet1813.85%450.00%
Shawn Bohrer96.92%112.50%
Paolo Abeni21.54%225.00%
Total130100.00%8100.00%


static __inline__ void udpv6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, u8 type, u8 code, int offset, __be32 info) { __udp6_lib_err(skb, opt, type, code, offset, info, &udp_table); }

Contributors

PersonTokensPropCommitsCommitProp
Gerrit Renker4085.11%116.67%
Brian Haley24.26%116.67%
Eric Dumazet24.26%116.67%
Al Viro12.13%116.67%
Linus Torvalds (pre-git)12.13%116.67%
Hideaki Yoshifuji / 吉藤英明12.13%116.67%
Total47100.00%6100.00%

static struct static_key udpv6_encap_needed __read_mostly;
void udpv6_encap_enable(void) { static_key_enable(&udpv6_encap_needed); }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin LaHaise1292.31%150.00%
Paolo Bonzini17.69%150.00%
Total13100.00%2100.00%

EXPORT_SYMBOL(udpv6_encap_enable);
static int udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) { struct udp_sock *up = udp_sk(sk); int is_udplite = IS_UDPLITE(sk); if (!xfrm6_policy_check