cregit-Linux how code gets into the kernel

Release 4.14 net/ipv4/ip_output.c

Directory: net/ipv4
 * INET         An implementation of the TCP/IP protocol suite for the LINUX
 *              operating system.  INET is implemented using the  BSD Socket
 *              interface as the means of communication with the user level.
 *              The Internet Protocol (IP) output module.
 * Authors:     Ross Biro
 *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *              Donald Becker, <>
 *              Alan Cox, <>
 *              Richard Underwood
 *              Stefan Becker, <>
 *              Jorge Cwik, <>
 *              Arnt Gulbrandsen, <>
 *              Hirokazu Takahashi, <>
 *      See ip_input.c for original log
 *      Fixes:
 *              Alan Cox        :       Missing nonblock feature in ip_build_xmit.
 *              Mike Kilburn    :       htons() missing in ip_build_xmit.
 *              Bradford Johnson:       Fix faulty handling of some frames when
 *                                      no route is found.
 *              Alexander Demenshin:    Missing sk/skb free in ip_queue_xmit
 *                                      (in case if packet not accepted by
 *                                      output firewall rules)
 *              Mike McLagan    :       Routing by source
 *              Alexey Kuznetsov:       use new route cache
 *              Andi Kleen:             Fix broken PMTU recovery and remove
 *                                      some redundant tests.
 *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
 *              Andi Kleen      :       Replace ip_reply with ip_send_reply.
 *              Andi Kleen      :       Split fast and slow ip_build_xmit path
 *                                      for decreased register pressure on x86
 *                                      and more readibility.
 *              Marc Boucher    :       When call_out_firewall returns FW_QUEUE,
 *                                      silently drop skb instead of failing with -EPERM.
 *              Detlev Wengorz  :       Copy protocol for fragments.
 *              Hirokazu Takahashi:     HW checksumming for outgoing UDP
 *                                      datagrams.
 *              Hirokazu Takahashi:     sendfile() on UDP works now.

#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/highmem.h>
#include <linux/slab.h>

#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/proc_fs.h>
#include <linux/stat.h>
#include <linux/init.h>

#include <net/snmp.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/route.h>
#include <net/xfrm.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <net/arp.h>
#include <net/icmp.h>
#include <net/checksum.h>
#include <net/inetpeer.h>
#include <net/lwtunnel.h>
#include <linux/bpf-cgroup.h>
#include <linux/igmp.h>
#include <linux/netfilter_ipv4.h>
#include <linux/netfilter_bridge.h>
#include <linux/netlink.h>
#include <linux/tcp.h>

static int
ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
	    unsigned int mtu,
	    int (*output)(struct net *, struct sock *, struct sk_buff *));

/* Generate a checksum for an outgoing IP datagram. */

void ip_send_check(struct iphdr *iph) { iph->check = 0; iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); }


Linus Torvalds (pre-git)34100.00%1100.00%

int __ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb) { struct iphdr *iph = ip_hdr(skb); iph->tot_len = htons(skb->len); ip_send_check(iph); /* if egress device is enslaved to an L3 master device pass the * skb to its handler for processing */ skb = l3mdev_ip_out(sk, skb); if (unlikely(!skb)) return 0; skb->protocol = htons(ETH_P_IP); return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT, net, sk, skb, NULL, skb_dst(skb)->dev, dst_output); }


Herbert Xu4746.53%19.09%
David Ahern2120.79%19.09%
David S. Miller109.90%19.09%
Eli Cooper98.91%19.09%
Eric W. Biedermann98.91%436.36%
Eric Dumazet32.97%19.09%
Jan Engelhardt10.99%19.09%
Patrick McHardy10.99%19.09%

int ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb) { int err; err = __ip_local_out(net, sk, skb); if (likely(err == 1)) err = dst_output(net, sk, skb); return err; }


Herbert Xu3663.16%111.11%
Eric W. Biedermann1424.56%777.78%
Eric Dumazet712.28%111.11%

static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst) { int ttl = inet->uc_ttl; if (ttl < 0) ttl = ip4_dst_hoplimit(dst); return ttl; }


David S. Miller3997.50%266.67%
Arnaldo Carvalho de Melo12.50%133.33%

/* * Add an ip header to a skbuff and send it out. * */
int ip_build_and_send_pkt(struct sk_buff *skb, const struct sock *sk, __be32 saddr, __be32 daddr, struct ip_options_rcu *opt) { struct inet_sock *inet = inet_sk(sk); struct rtable *rt = skb_rtable(skb); struct net *net = sock_net(sk); struct iphdr *iph; /* Build the IP header. */ skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0)); skb_reset_network_header(skb); iph = ip_hdr(skb); iph->version = 4; iph->ihl = 5; iph->tos = inet->tos; iph->ttl = ip_select_ttl(inet, &rt->dst); iph->daddr = (opt && opt->opt.srr ? opt->opt.faddr : daddr); iph->saddr = saddr; iph->protocol = sk->sk_protocol; if (ip_dont_fragment(sk, &rt->dst)) { iph->frag_off = htons(IP_DF); iph->id = 0; } else { iph->frag_off = 0; __ip_select_ident(net, iph, 1); } if (opt && opt->opt.optlen) { iph->ihl += opt->opt.optlen>>2; ip_options_build(skb, &opt->opt, daddr, rt, 0); } skb->priority = sk->sk_priority; if (!skb->mark) skb->mark = sk->sk_mark; /* Send it out. */ return ip_local_out(net, skb->sk, skb); }


Linus Torvalds (pre-git)14148.45%932.14%
Eric Dumazet5518.90%310.71%
David S. Miller3512.03%310.71%
Arnaldo Carvalho de Melo175.84%414.29%
Eric W. Biedermann175.84%310.71%
Tóth László Attila82.75%13.57%
Jamal Hadi Salim72.41%13.57%
Alexey Kuznetsov72.41%13.57%
Al Viro20.69%13.57%
Herbert Xu10.34%13.57%
Hannes Frederic Sowa10.34%13.57%

static int ip_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb) { struct dst_entry *dst = skb_dst(skb); struct rtable *rt = (struct rtable *)dst; struct net_device *dev = dst->dev; unsigned int hh_len = LL_RESERVED_SPACE(dev); struct neighbour *neigh; u32 nexthop; if (rt->rt_type == RTN_MULTICAST) { IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTMCAST, skb->len); } else if (rt->rt_type == RTN_BROADCAST) IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTBCAST, skb->len); /* Be paranoid, rather than too clever. */ if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { struct sk_buff *skb2; skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev)); if (!skb2) { kfree_skb(skb); return -ENOMEM; } if (skb->sk) skb_set_owner_w(skb2, skb->sk); consume_skb(skb); skb = skb2; } if (lwtunnel_xmit_redirect(dst->lwtstate)) { int res = lwtunnel_xmit(skb); if (res < 0 || res == LWTUNNEL_XMIT_DONE) return res; } rcu_read_lock_bh(); nexthop = (__force u32) rt_nexthop(rt, ip_hdr(skb)->daddr); neigh = __ipv4_neigh_lookup_noref(dev, nexthop); if (unlikely(!neigh)) neigh = __neigh_create(&arp_tbl, &nexthop, dev, false); if (!IS_ERR(neigh)) { int res; sock_confirm_neigh(skb, neigh); res = neigh_output(neigh, skb); rcu_read_unlock_bh(); return res; } rcu_read_unlock_bh(); net_dbg_ratelimited("%s: No header cache and no neighbour!\n", __func__); kfree_skb(skb); return -EINVAL; }


Alexey Kuznetsov8927.38%26.90%
David S. Miller6219.08%620.69%
Linus Torvalds (pre-git)3711.38%413.79%
Mitsuru Chinen3510.77%13.45%
Roopa Prabhu329.85%13.45%
Julian Anastasov206.15%310.34%
Eric Dumazet164.92%310.34%
Neil Horman144.31%13.45%
Eric W. Biedermann72.15%26.90%
Vasiliy Kulikov41.23%13.45%
Joe Perches41.23%13.45%
Pavel Emelyanov20.62%13.45%
Ian Morris10.31%13.45%
Stephen Hemminger10.31%13.45%
Chuck Lever10.31%13.45%

static int ip_finish_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb, unsigned int mtu) { netdev_features_t features; struct sk_buff *segs; int ret = 0; /* common case: seglen is <= mtu */ if (skb_gso_validate_mtu(skb, mtu)) return ip_finish_output2(net, sk, skb); /* Slowpath - GSO segment length exceeds the egress MTU. * * This can happen in several cases: * - Forwarding of a TCP GRO skb, when DF flag is not set. * - Forwarding of an skb that arrived on a virtualization interface * (virtio-net/vhost/tap) with TSO/GSO size set by other network * stack. * - Local GSO skb transmitted on an NETIF_F_TSO tunnel stacked over an * interface with a smaller MTU. * - Arriving GRO skb (or GSO skb in a virtualized environment) that is * bridged to a NETIF_F_TSO tunnel stacked over an interface with an * insufficent MTU. */ features = netif_skb_features(skb); BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_SGO_CB_OFFSET); segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK); if (IS_ERR_OR_NULL(segs)) { kfree_skb(skb); return -ENOMEM; } consume_skb(skb); do { struct sk_buff *nskb = segs->next; int err; segs->next = NULL; err = ip_fragment(net, sk, segs, mtu, ip_finish_output2); if (err && ret == 0) ret = err; segs = nskb; } while (segs); return ret; }


Florian Westphal13878.86%333.33%
Konstantin Khlebnikov148.00%111.11%
Eric W. Biedermann95.14%111.11%
David S. Miller95.14%111.11%
Marcelo Ricardo Leitner31.71%111.11%
Lance Richardson21.14%222.22%

static int ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) { unsigned int mtu; int ret; ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb); if (ret) { kfree_skb(skb); return ret; } #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM) /* Policy lookup after SNAT yielded a new policy */ if (skb_dst(skb)->xfrm) { IPCB(skb)->flags |= IPSKB_REROUTED; return dst_output(net, sk, skb); } #endif mtu = ip_skb_dst_mtu(sk, skb); if (skb_is_gso(skb)) return ip_finish_output_gso(net, sk, skb, mtu); if (skb->len > mtu || (IPCB(skb)->flags & IPSKB_FRAG_PMTU)) return ip_fragment(net, sk, skb, mtu, ip_finish_output2); return ip_finish_output2(net, sk, skb); }


Patrick McHardy4828.92%317.65%
Florian Westphal4024.10%317.65%
Daniel Mack2615.66%15.88%
Linus Torvalds (pre-git)1911.45%211.76%
Eric W. Biedermann148.43%423.53%
David S. Miller137.83%15.88%
Eric Dumazet31.81%15.88%
Shmulik Ladkani21.20%15.88%
Adrian Bunk10.60%15.88%

static int ip_mc_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) { int ret; ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb); if (ret) { kfree_skb(skb); return ret; } return dev_loopback_xmit(net, sk, skb); }


Daniel Mack57100.00%1100.00%

int ip_mc_output(struct net *net, struct sock *sk, struct sk_buff *skb) { struct rtable *rt = skb_rtable(skb); struct net_device *dev = rt->; /* * If the indicated interface is up and running, send the packet. */ IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len); skb->dev = dev; skb->protocol = htons(ETH_P_IP); /* * Multicasts are looped back for other local users */ if (rt->rt_flags&RTCF_MULTICAST) { if (sk_mc_loop(sk) #ifdef CONFIG_IP_MROUTE /* Small optimization: do not loopback not local frames, which returned after forwarding; they will be dropped by ip_mr_input in any case. Note, that local frames are looped back to be delivered to local recipients. This check is duplicated in ip_mr_input at the moment. */ && ((rt->rt_flags & RTCF_LOCAL) || !(IPCB(skb)->flags & IPSKB_FORWARDED)) #endif ) { struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); if (newskb) NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING, net, sk, newskb, NULL, newskb->dev, ip_mc_finish_output); } /* Multicasts with ttl 0 must not go beyond the host */ if (ip_hdr(skb)->ttl == 0) { kfree_skb(skb); return 0; } } if (rt->rt_flags&RTCF_BROADCAST) { struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); if (newskb) NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING, net, sk, newskb, NULL, newskb->dev, ip_mc_finish_output); } return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, net, sk, skb, NULL, skb->dev, ip_finish_output, !(IPCB(skb)->flags & IPSKB_REROUTED)); }


Linus Torvalds (pre-git)17267.19%1135.48%
Patrick McHardy249.38%39.68%
Eric W. Biedermann155.86%39.68%
David S. Miller83.12%26.45%
Eric Dumazet83.12%26.45%
Linus Torvalds83.12%13.23%
Neil Horman51.95%13.23%
Alexey Kuznetsov41.56%13.23%
Jan Engelhardt31.17%13.23%
Arnaldo Carvalho de Melo31.17%13.23%
Daniel Mack20.78%13.23%
Hideaki Yoshifuji / 吉藤英明10.39%13.23%
Octavian Purdila10.39%13.23%
Herbert Xu10.39%13.23%
Pavel Emelyanov10.39%13.23%

int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb) { struct net_device *dev = skb_dst(skb)->dev; IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len); skb->dev = dev; skb->protocol = htons(ETH_P_IP); return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, net, sk, skb, NULL, dev, ip_finish_output, !(IPCB(skb)->flags & IPSKB_REROUTED)); }


Patrick McHardy3842.22%316.67%
Linus Torvalds (pre-git)1516.67%316.67%
Eric W. Biedermann1011.11%316.67%
Eric Dumazet88.89%211.11%
Alexey Kuznetsov77.78%15.56%
Neil Horman55.56%15.56%
Ananda Raju22.22%15.56%
David S. Miller22.22%15.56%
Jan Engelhardt11.11%15.56%
Pavel Emelyanov11.11%15.56%
Herbert Xu11.11%15.56%

/* * copy saddr and daddr, possibly using 64bit load/stores * Equivalent to : * iph->saddr = fl4->saddr; * iph->daddr = fl4->daddr; */
static void ip_copy_addrs(struct iphdr *iph, const struct flowi4 *fl4) { BUILD_BUG_ON(offsetof(typeof(*fl4), daddr) != offsetof(typeof(*fl4), saddr) + sizeof(fl4->saddr)); memcpy(&iph->saddr, &fl4->saddr, sizeof(fl4->saddr) + sizeof(fl4->daddr)); }


Eric Dumazet76100.00%1100.00%

/* Note: skb->sk can be different from sk, in case of tunnels */
int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl) { struct inet_sock *inet = inet_sk(sk); struct net *net = sock_net(sk); struct ip_options_rcu *inet_opt; struct flowi4 *fl4; struct rtable *rt; struct iphdr *iph; int res; /* Skip all of this if the packet is already routed, * f.e. by something like SCTP. */ rcu_read_lock(); inet_opt = rcu_dereference(inet->inet_opt); fl4 = &fl->u.ip4; rt = skb_rtable(skb); if (rt) goto packet_routed; /* Make sure we can route this packet. */ rt = (struct rtable *)__sk_dst_check(sk, 0); if (!rt) { __be32 daddr; /* Use correct destination address if we have options. */ daddr = inet->inet_daddr; if (inet_opt && inet_opt->opt.srr) daddr = inet_opt->opt.faddr; /* If this fails, retransmit mechanism of transport layer will * keep trying until route appears or the connection times * itself out. */ rt = ip_route_output_ports(net, fl4, sk, daddr, inet->inet_saddr, inet->inet_dport, inet->inet_sport, sk->sk_protocol, RT_CONN_FLAGS(sk), sk->sk_bound_dev_if); if (IS_ERR(rt)) goto no_route; sk_setup_caps(sk, &rt->dst); } skb_dst_set_noref(skb, &rt->dst); packet_routed: if (inet_opt && inet_opt->opt.is_strictroute && rt->rt_uses_gateway) goto no_route; /* OK, we know where to send it, allocate and build IP header. */ skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0)); skb_reset_network_header(skb); iph = ip_hdr(skb); *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff)); if (ip_dont_fragment(sk, &rt->dst) && !skb->ignore_df) iph->frag_off = htons(IP_DF); else iph->frag_off = 0; iph->ttl = ip_select_ttl(inet, &rt->dst); iph->protocol = sk->sk_protocol; ip_copy_addrs(iph, fl4); /* Transport layer set skb-> itself. */ if (inet_opt && inet_opt->opt.optlen) { iph->ihl += inet_opt->opt.optlen >> 2; ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0); } ip_select_ident_segs(net, skb, sk, skb_shinfo(skb)->gso_segs ?: 1); /* TODO : should we use skb->sk here instead of sk ? */ skb->priority = sk->sk_priority; skb->mark = sk->sk_mark; res = ip_local_out(net, sk, skb); rcu_read_unlock(); return res; no_route: rcu_read_unlock(); IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); kfree_skb(skb); return -EHOSTUNREACH; }


Alexey Kuznetsov20642.56%35.56%
Linus Torvalds (pre-git)7615.70%47.41%
Eric Dumazet7315.08%814.81%
David S. Miller6212.81%1018.52%
Eric W. Biedermann183.72%47.41%
Arnaldo Carvalho de Melo163.31%59.26%
Tóth László Attila81.65%11.85%
Herbert Xu40.83%23.70%
Hideaki Yoshifuji / 吉藤英明30.62%23.70%
Al Viro20.41%23.70%
Shan Wei20.41%11.85%
Venkat Yekkirala20.41%11.85%
Sridhar Samudrala20.41%11.85%
Denis V. Lunev20.41%23.70%
Ansis Atteka10.21%11.85%
Julian Anastasov10.21%11.85%
Ian Morris10.21%11.85%
Hannes Frederic Sowa10.21%11.85%
Atis Elsts10.21%11.85%
Américo Wang10.21%11.85%
Pavel Emelyanov10.21%11.85%
Linus Torvalds10.21%11.85%

static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from) { to->pkt_type = from->pkt_type; to->priority = from->priority; to->protocol = from->protocol; skb_dst_drop(to); skb_dst_copy(to, from); to->dev = from->dev; to->mark = from->mark; /* Copy the flags to each fragment. */ IPCB(to)->flags = IPCB(from)->flags; #ifdef CONFIG_NET_SCHED to->tc_index = from->tc_index; #endif nf_copy(to, from); #if IS_ENABLED(CONFIG_IP_VS) to->ipvs_property = from->ipvs_property; #endif skb_copy_secmark(to, from); }


Alexey Kuznetsov8265.08%19.09%
Julian Anastasov1411.11%19.09%
Thomas Graf86.35%19.09%
James Morris75.56%19.09%
Eric Dumazet53.97%218.18%
Patrick McHardy43.17%19.09%
Jozsef Kadlecsik32.38%218.18%
Yasuyuki Kozakai21.59%19.09%
Javier Martinez Canillas10.79%19.09%

static int ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, unsigned int mtu, int (*output)(struct net *, struct sock *, struct sk_buff *)) { struct iphdr *iph = ip_hdr(skb); if ((iph->frag_off & htons(IP_DF)) == 0) return ip_do_fragment(net, sk, skb, output); if (unlikely(!skb->ignore_df || (IPCB(skb)->frag_max_size && IPCB(skb)->frag_max_size > mtu))) { IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS); icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu)); kfree_skb(skb); return -EMSGSIZE; } return ip_do_fragment(net, sk, skb, output); }


Alexey Kuznetsov6743.79%214.29%
Florian Westphal2113.73%214.29%
Patrick McHardy1811.76%17.14%
Eric W. Biedermann149.15%214.29%
Andy Zhou138.50%17.14%
David S. Miller95.88%17.14%
Wei Dong53.27%17.14%
Eric Dumazet21.31%17.14%
Hannes Frederic Sowa21.31%17.14%
Pavel Emelyanov10.65%17.14%
Américo Wang10.65%17.14%

/* * This IP datagram is too large to be sent in one piece. Break it up into * smaller pieces (each of size equal to IP header plus * a block of the data of the original IP data part) that will yet fit in a * single device frame, and queue such a frame for sending. */
int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, int (*output)(struct net *, struct sock *, struct sk_buff *)) { struct iphdr *iph; int ptr; struct sk_buff *skb2; unsigned int mtu, hlen, left, len, ll_rs; int offset; __be16 not_last_frag; struct rtable *rt = skb_rtable(skb); int err = 0; /* for offloaded checksums cleanup checksum before fragmentation */ if (skb->ip_summed == CHECKSUM_PARTIAL && (err = skb_checksum_help(skb))) goto fail; /* * Point into the IP datagram header. */ iph = ip_hdr(skb); mtu = ip_skb_dst_mtu(sk, skb); if (IPCB(skb)->frag_max_size && IPCB(skb)->frag_max_size < mtu) mtu = IPCB(skb)->frag_max_size; /* * Setup starting values. */ hlen = iph->ihl * 4; mtu = mtu - hlen; /* Size of data space */ IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE; ll_rs = LL_RESERVED_SPACE(rt->