cregit-Linux how code gets into the kernel

Release 4.11 net/ipv6/udp.c

Directory: net/ipv6
/*
 *      UDP over IPv6
 *      Linux INET6 implementation
 *
 *      Authors:
 *      Pedro Roque             <roque@di.fc.ul.pt>
 *
 *      Based on linux/ipv4/udp.c
 *
 *      Fixes:
 *      Hideaki YOSHIFUJI       :       sin6_scope_id support
 *      YOSHIFUJI Hideaki @USAGI and:   Support IPV6_V6ONLY socket option, which
 *      Alexey Kuznetsov                allow both IPv4 and IPv6 sockets to bind
 *                                      a single port at the same time.
 *      Kazunori MIYAZAWA @USAGI:       change process style to use ip6_append_data
 *      YOSHIFUJI Hideaki @USAGI:       convert /proc/net/udp6 to seq_file.
 *
 *      This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 */

#include <linux/errno.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/in6.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/ipv6.h>
#include <linux/icmpv6.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/uaccess.h>

#include <net/addrconf.h>
#include <net/ndisc.h>
#include <net/protocol.h>
#include <net/transp_v6.h>
#include <net/ip6_route.h>
#include <net/raw.h>
#include <net/tcp_states.h>
#include <net/ip6_checksum.h>
#include <net/xfrm.h>
#include <net/inet6_hashtables.h>
#include <net/busy_poll.h>
#include <net/sock_reuseport.h>

#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <trace/events/skb.h>
#include "udp_impl.h"


static bool udp6_lib_exact_dif_match(struct net *net, struct sk_buff *skb) { #if defined(CONFIG_NET_L3_MASTER_DEV) if (!net->ipv4.sysctl_udp_l3mdev_accept && skb && ipv6_l3mdev_skb(IP6CB(skb)->flags)) return true; #endif return false; }

Contributors

PersonTokensPropCommitsCommitProp
Robert Shearman51100.00%1100.00%
Total51100.00%1100.00%


static u32 udp6_ehashfn(const struct net *net, const struct in6_addr *laddr, const u16 lport, const struct in6_addr *faddr, const __be16 fport) { static u32 udp6_ehash_secret __read_mostly; static u32 udp_ipv6_hash_secret __read_mostly; u32 lhash, fhash; net_get_random_once(&udp6_ehash_secret, sizeof(udp6_ehash_secret)); net_get_random_once(&udp_ipv6_hash_secret, sizeof(udp_ipv6_hash_secret)); lhash = (__force u32)laddr->s6_addr32[3]; fhash = __ipv6_addr_jhash(faddr, udp_ipv6_hash_secret); return __inet6_ehashfn(lhash, lport, fhash, fport, udp_ipv6_hash_secret + net_hash_mix(net)); }

Contributors

PersonTokensPropCommitsCommitProp
Hannes Frederic Sowa10797.27%250.00%
Eric Dumazet21.82%125.00%
Vladislav Yasevich10.91%125.00%
Total110100.00%4100.00%


static u32 udp6_portaddr_hash(const struct net *net, const struct in6_addr *addr6, unsigned int port) { unsigned int hash, mix = net_hash_mix(net); if (ipv6_addr_any(addr6)) hash = jhash_1word(0, mix); else if (ipv6_addr_v4mapped(addr6)) hash = jhash_1word((__force u32)addr6->s6_addr32[3], mix); else hash = jhash2((__force u32 *)addr6->s6_addr32, 4, mix); return hash ^ port; }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet9898.99%375.00%
Brian Haley11.01%125.00%
Total99100.00%4100.00%


int udp_v6_get_port(struct sock *sk, unsigned short snum) { unsigned int hash2_nulladdr = udp6_portaddr_hash(sock_net(sk), &in6addr_any, snum); unsigned int hash2_partial = udp6_portaddr_hash(sock_net(sk), &sk->sk_v6_rcv_saddr, 0); /* precompute partial secondary hash */ udp_sk(sk)->udp_portaddr_hash = hash2_partial; return udp_lib_get_port(sk, snum, hash2_nulladdr); }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet4767.14%327.27%
Linus Torvalds (pre-git)1724.29%436.36%
Arnaldo Carvalho de Melo45.71%218.18%
Pavel Emelyanov11.43%19.09%
Gerrit Renker11.43%19.09%
Total70100.00%11100.00%


static void udp_v6_rehash(struct sock *sk) { u16 new_hash = udp6_portaddr_hash(sock_net(sk), &sk->sk_v6_rcv_saddr, inet_sk(sk)->inet_num); udp_lib_rehash(sk, new_hash); }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet41100.00%2100.00%
Total41100.00%2100.00%


static int compute_score(struct sock *sk, struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, unsigned short hnum, int dif, bool exact_dif) { int score; struct inet_sock *inet; if (!net_eq(sock_net(sk), net) || udp_sk(sk)->udp_port_hash != hnum || sk->sk_family != PF_INET6) return -1; score = 0; inet = inet_sk(sk); if (inet->inet_dport) { if (inet->inet_dport != sport) return -1; score++; } if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) { if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return -1; score++; } if (!ipv6_addr_any(&sk->sk_v6_daddr)) { if (!ipv6_addr_equal(&sk->sk_v6_daddr, saddr)) return -1; score++; } if (sk->sk_bound_dev_if || exact_dif) { if (sk->sk_bound_dev_if != dif) return -1; score++; } if (sk->sk_incoming_cpu == raw_smp_processor_id()) score++; return score; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)10949.32%419.05%
Eric Dumazet5826.24%628.57%
Joe Perches198.60%14.76%
Hideaki Yoshifuji / 吉藤英明114.98%314.29%
Pavel Emelyanov83.62%14.76%
Robert Shearman52.26%14.76%
Arnaldo Carvalho de Melo31.36%14.76%
Xuemin Su31.36%14.76%
Balazs Scheidler20.90%14.76%
David S. Miller20.90%14.76%
Gerrit Renker10.45%14.76%
Total221100.00%21100.00%

/* called with rcu_read_lock() */
static struct sock *udp6_lib_lookup2(struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, unsigned int hnum, int dif, bool exact_dif, struct udp_hslot *hslot2, struct sk_buff *skb) { struct sock *sk, *result; int score, badness, matches = 0, reuseport = 0; u32 hash = 0; result = NULL; badness = -1; udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { score = compute_score(sk, net, saddr, sport, daddr, hnum, dif, exact_dif); if (score > badness) { reuseport = sk->sk_reuseport; if (reuseport) { hash = udp6_ehashfn(net, daddr, hnum, saddr, sport); result = reuseport_select_sock(sk, hash, skb, sizeof(struct udphdr)); if (result) return result; matches = 1; } result = sk; badness = score; } else if (score == badness && reuseport) { matches++; if (reciprocal_scale(hash, matches) == 0) result = sk; hash = next_pseudo_random32(hash); } } return result; }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet11250.45%327.27%
Tom Herbert7332.88%19.09%
Craig Gallek2611.71%327.27%
Robert Shearman52.25%19.09%
Daniel Borkmann41.80%19.09%
Xuemin Su10.45%19.09%
Hannes Frederic Sowa10.45%19.09%
Total222100.00%11100.00%

/* rcu_read_lock() must be held */
struct sock *__udp6_lib_lookup(struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, __be16 dport, int dif, struct udp_table *udptable, struct sk_buff *skb) { struct sock *sk, *result; unsigned short hnum = ntohs(dport); unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask); struct udp_hslot *hslot2, *hslot = &udptable->hash[slot]; bool exact_dif = udp6_lib_exact_dif_match(net, skb); int score, badness, matches = 0, reuseport = 0; u32 hash = 0; if (hslot->count > 10) { hash2 = udp6_portaddr_hash(net, daddr, hnum); slot2 = hash2 & udptable->mask; hslot2 = &udptable->hash2[slot2]; if (hslot->count < hslot2->count) goto begin; result = udp6_lib_lookup2(net, saddr, sport, daddr, hnum, dif, exact_dif, hslot2, skb); if (!result) { unsigned int old_slot2 = slot2; hash2 = udp6_portaddr_hash(net, &in6addr_any, hnum); slot2 = hash2 & udptable->mask; /* avoid searching the same slot again. */ if (unlikely(slot2 == old_slot2)) return result; hslot2 = &udptable->hash2[slot2]; if (hslot->count < hslot2->count) goto begin; result = udp6_lib_lookup2(net, saddr, sport, daddr, hnum, dif, exact_dif, hslot2, skb); } return result; } begin: result = NULL; badness = -1; sk_for_each_rcu(sk, &hslot->head) { score = compute_score(sk, net, saddr, sport, daddr, hnum, dif, exact_dif); if (score > badness) { reuseport = sk->sk_reuseport; if (reuseport) { hash = udp6_ehashfn(net, daddr, hnum, saddr, sport); result = reuseport_select_sock(sk, hash, skb, sizeof(struct udphdr)); if (result) return result; matches = 1; } result = sk; badness = score; } else if (score == badness && reuseport) { matches++; if (reciprocal_scale(hash, matches) == 0) result = sk; hash = next_pseudo_random32(hash); } } return result; }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet27962.98%635.29%
Tom Herbert7216.25%15.88%
Craig Gallek296.55%317.65%
Xuemin Su214.74%15.88%
Linus Torvalds (pre-git)184.06%15.88%
Robert Shearman163.61%15.88%
Daniel Borkmann40.90%15.88%
Balazs Scheidler20.45%15.88%
Jorge Boncompte10.23%15.88%
Hannes Frederic Sowa10.23%15.88%
Total443100.00%17100.00%

EXPORT_SYMBOL_GPL(__udp6_lib_lookup);
static struct sock *__udp6_lib_lookup_skb(struct sk_buff *skb, __be16 sport, __be16 dport, struct udp_table *udptable) { const struct ipv6hdr *iph = ipv6_hdr(skb); struct sock *sk; sk = skb_steal_sock(skb); if (unlikely(sk)) return sk; return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport, &iph->daddr, dport, inet6_iif(skb), udptable, skb); }

Contributors

PersonTokensPropCommitsCommitProp
KOVACS Krisztian7582.42%228.57%
Ian Morris66.59%114.29%
Alexander Duyck55.49%114.29%
Eric Dumazet33.30%228.57%
Craig Gallek22.20%114.29%
Total91100.00%7100.00%


struct sock *udp6_lib_lookup_skb(struct sk_buff *skb, __be16 sport, __be16 dport) { const struct ipv6hdr *iph = ipv6_hdr(skb); return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport, &iph->daddr, dport, inet6_iif(skb), &udp_table, skb); }

Contributors

PersonTokensPropCommitsCommitProp
Tom Herbert6296.88%150.00%
Alexander Duyck23.12%150.00%
Total64100.00%2100.00%

EXPORT_SYMBOL_GPL(udp6_lib_lookup_skb); /* Must be called under rcu_read_lock(). * Does increment socket refcount. */ #if IS_ENABLED(CONFIG_NETFILTER_XT_MATCH_SOCKET) || \ IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TPROXY) || \ IS_ENABLED(CONFIG_NF_SOCKET_IPV6)
struct sock *udp6_lib_lookup(struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, __be16 dport, int dif) { struct sock *sk; sk = __udp6_lib_lookup(net, saddr, sport, daddr, dport, dif, &udp_table, NULL); if (sk && !atomic_inc_not_zero(&sk->sk_refcnt)) sk = NULL; return sk; }

Contributors

PersonTokensPropCommitsCommitProp
Balazs Scheidler4758.75%133.33%
Eric Dumazet3138.75%133.33%
Craig Gallek22.50%133.33%
Total80100.00%3100.00%

EXPORT_SYMBOL_GPL(udp6_lib_lookup); #endif /* * This should be easy, if there is something there we * return it, otherwise we block. */
int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock, int flags, int *addr_len) { struct ipv6_pinfo *np = inet6_sk(sk); struct inet_sock *inet = inet_sk(sk); struct sk_buff *skb; unsigned int ulen, copied; int peeked, peeking, off; int err; int is_udplite = IS_UDPLITE(sk); bool checksum_valid = false; int is_udp4; if (flags & MSG_ERRQUEUE) return ipv6_recv_error(sk, msg, len, addr_len); if (np->rxpmtu && np->rxopt.bits.rxpmtu) return ipv6_recv_rxpmtu(sk, msg, len, addr_len); try_again: peeking = off = sk_peek_offset(sk, flags); skb = __skb_recv_udp(sk, flags, noblock, &peeked, &off, &err); if (!skb) return err; ulen = skb->len; copied = len; if (copied > ulen - off) copied = ulen - off; else if (copied < ulen) msg->msg_flags |= MSG_TRUNC; is_udp4 = (skb->protocol == htons(ETH_P_IP)); /* * If checksum is needed at all, try to do it while copying the * data. If the data is truncated, or if we only want a partial * coverage checksum (UDP-Lite), do it before the copy. */ if (copied < ulen || peeking || (is_udplite && UDP_SKB_CB(skb)->partial_cov)) { checksum_valid = !udp_lib_checksum_complete(skb); if (!checksum_valid) goto csum_copy_err; } if (checksum_valid || skb_csum_unnecessary(skb)) err = skb_copy_datagram_msg(skb, off, msg, copied); else { err = skb_copy_and_csum_datagram_msg(skb, off, msg); if (err == -EINVAL) goto csum_copy_err; } if (unlikely(err)) { if (!peeked) { atomic_inc(&sk->sk_drops); if (is_udp4) UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); else UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); } kfree_skb(skb); return err; } if (!peeked) { if (is_udp4) UDP_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS, is_udplite); else UDP6_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS, is_udplite); } sock_recv_ts_and_drops(msg, sk, skb); /* Copy the address. */ if (msg->msg_name) { DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name); sin6->sin6_family = AF_INET6; sin6->sin6_port = udp_hdr(skb)->source; sin6->sin6_flowinfo = 0; if (is_udp4) { ipv6_addr_set_v4mapped(ip_hdr(skb)->saddr, &sin6->sin6_addr); sin6->sin6_scope_id = 0; } else { sin6->sin6_addr = ipv6_hdr(skb)->saddr; sin6->sin6_scope_id = ipv6_iface_scope_id(&sin6->sin6_addr, inet6_iif(skb)); } *addr_len = sizeof(*sin6); } if (np->rxopt.all) ip6_datagram_recv_common_ctl(sk, msg, skb); if (is_udp4) { if (inet->cmsg_flags) ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off); } else { if (np->rxopt.all) ip6_datagram_recv_specific_ctl(sk, msg, skb); } err = copied; if (flags & MSG_TRUNC) err = ulen; skb_consume_udp(sk, skb, peeking ? -err : err); return err; csum_copy_err: if (!__sk_queue_drop_skb(sk, skb, flags, udp_skb_destructor)) { if (is_udp4) { UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); } else { UDP6_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); } } kfree_skb(skb); /* starting over for a new packet, but check if we need to yield */ cond_resched(); msg->msg_flags &= ~MSG_TRUNC; goto try_again; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)21630.29%1016.95%
Eric Dumazet11716.41%915.25%
Wei Yongjun547.57%23.39%
Herbert Xu496.87%58.47%
Hideaki Yoshifuji / 吉藤英明496.87%23.39%
Hannes Frederic Sowa466.45%46.78%
samanthakumar365.05%11.69%
Brian Haley304.21%23.39%
David S. Miller294.07%46.78%
Pavel Emelyanov152.10%23.39%
Gerrit Renker111.54%11.69%
Paolo Abeni91.26%35.08%
Arnaldo Carvalho de Melo91.26%35.08%
Samuel Thibault91.26%11.69%
Xufeng Zhang70.98%11.69%
Wang Chen60.84%11.69%
Steffen Hurrle60.84%11.69%
Linus Torvalds50.70%11.69%
Stephen Hemminger30.42%11.69%
Hideo Aoki30.42%11.69%
Duan Jiong10.14%11.69%
Neil Horman10.14%11.69%
Alexey Dobriyan10.14%11.69%
Al Viro10.14%11.69%
Total713100.00%59100.00%


void __udp6_lib_err(struct sk_buff *skb, struct inet6_skb_parm *opt, u8 type, u8 code, int offset, __be32 info, struct udp_table *udptable) { struct ipv6_pinfo *np; const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data; const struct in6_addr *saddr = &hdr->saddr; const struct in6_addr *daddr = &hdr->daddr; struct udphdr *uh = (struct udphdr *)(skb->data+offset); struct sock *sk; int harderr; int err; struct net *net = dev_net(skb->dev); sk = __udp6_lib_lookup(net, daddr, uh->dest, saddr, uh->source, inet6_iif(skb), udptable, skb); if (!sk) { __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev), ICMP6_MIB_INERRORS); return; } harderr = icmpv6_err_convert(type, code, &err); np = inet6_sk(sk); if (type == ICMPV6_PKT_TOOBIG) { if (!ip6_sk_accept_pmtu(sk)) goto out; ip6_sk_update_pmtu(skb, sk, info); if (np->pmtudisc != IPV6_PMTUDISC_DONT) harderr = 1; } if (type == NDISC_REDIRECT) { ip6_sk_redirect(skb, sk); goto out; } if (!np->recverr) { if (!harderr || sk->sk_state != TCP_ESTABLISHED) goto out; } else { ipv6_icmp_error(sk, skb, err, uh->dest, ntohl(info), (u8 *)(uh+1)); } sk->sk_err = err; sk->sk_error_report(sk); out: return; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)12640.26%725.00%
Wei Wang4213.42%13.57%
David S. Miller3410.86%310.71%
Linus Torvalds3310.54%13.57%
Duan Jiong319.90%27.14%
Hannes Frederic Sowa134.15%13.57%
Eric Dumazet82.56%414.29%
Hideaki Yoshifuji / 吉藤英明72.24%27.14%
Gerrit Renker61.92%13.57%
Pavel Emelyanov41.28%13.57%
Arnaldo Carvalho de Melo30.96%13.57%
Brian Haley20.64%13.57%
Craig Gallek20.64%13.57%
Ian Morris10.32%13.57%
Al Viro10.32%13.57%
Total313100.00%28100.00%


int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) { int rc; if (!ipv6_addr_any(&sk->sk_v6_daddr)) { sock_rps_save_rxhash(sk, skb); sk_mark_napi_id(sk, skb); sk_incoming_cpu_update(sk); } else { sk_mark_napi_id_once(sk, skb); } rc = __udp_enqueue_schedule_skb(sk, skb); if (rc < 0) { int is_udplite = IS_UDPLITE(sk); /* Note that an ENOMEM error is charged twice */ if (rc == -ENOMEM) UDP6_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS, is_udplite); UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); kfree_skb(skb); return -1; } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin LaHaise10178.29%114.29%
Eric Dumazet1813.95%457.14%
Shawn Bohrer96.98%114.29%
Paolo Abeni10.78%114.29%
Total129100.00%7100.00%


static __inline__ void udpv6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, u8 type, u8 code, int offset, __be32 info) { __udp6_lib_err(skb, opt, type, code, offset, info, &udp_table); }

Contributors

PersonTokensPropCommitsCommitProp
Gerrit Renker4085.11%116.67%
Eric Dumazet24.26%116.67%
Brian Haley24.26%116.67%
Linus Torvalds (pre-git)12.13%116.67%
Al Viro12.13%116.67%
Hideaki Yoshifuji / 吉藤英明12.13%116.67%
Total47100.00%6100.00%

static struct static_key udpv6_encap_needed __read_mostly;
void udpv6_encap_enable(void) { if (!static_key_enabled(&udpv6_encap_needed)) static_key_slow_inc(&udpv6_encap_needed); }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin LaHaise22100.00%1100.00%
Total22100.00%1100.00%

EXPORT_SYMBOL(udpv6_encap_enable);
int udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) { struct udp_sock *up = udp_sk(sk); int is_udplite = IS_UDPLITE(sk); if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) goto drop; if (static_key_false(&udpv6_encap_needed) && up->encap_type) { int (*encap_rcv)(struct sock *sk, struct sk_buff *skb); /* * This is an encapsulation socket so pass the skb to * the socket's udp_encap_rcv() hook. Otherwise, just * fall through and pass this up the UDP socket. * up->encap_rcv() returns the following value: * =0 if skb was successfully passed to the encap * handler or was discarded by it. * >0 if skb should be passed on to UDP. * <0 if skb should be resubmitted as proto -N */ /* if we're overly short, let UDP handle it */ encap_rcv = ACCESS_ONCE(up->encap_rcv); if (encap_rcv) { int ret; /* Verify checksum before giving to encap */ if (udp_lib_checksum_complete(skb)) goto csum_error; ret = encap_rcv(sk, skb); if (ret <= 0) { __UDP_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS, is_udplite); return -