cregit-Linux how code gets into the kernel

Release 4.11 net/sunrpc/xprtsock.c

Directory: net/sunrpc
/*
 * linux/net/sunrpc/xprtsock.c
 *
 * Client-side transport implementation for sockets.
 *
 * TCP callback races fixes (C) 1998 Red Hat
 * TCP send fixes (C) 1998 Red Hat
 * TCP NFS related read + write fixes
 *  (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
 *
 * Rewrite of larges part of the code in order to stabilize TCP stuff.
 * Fix behaviour when socket buffer is full.
 *  (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no>
 *
 * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com>
 *
 * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005.
 *   <gilles.quillard@bull.net>
 */

#include <linux/types.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/capability.h>
#include <linux/pagemap.h>
#include <linux/errno.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/net.h>
#include <linux/mm.h>
#include <linux/un.h>
#include <linux/udp.h>
#include <linux/tcp.h>
#include <linux/sunrpc/clnt.h>
#include <linux/sunrpc/addr.h>
#include <linux/sunrpc/sched.h>
#include <linux/sunrpc/svcsock.h>
#include <linux/sunrpc/xprtsock.h>
#include <linux/file.h>
#ifdef CONFIG_SUNRPC_BACKCHANNEL
#include <linux/sunrpc/bc_xprt.h>
#endif

#include <net/sock.h>
#include <net/checksum.h>
#include <net/udp.h>
#include <net/tcp.h>

#include <trace/events/sunrpc.h>

#include "sunrpc.h"

static void xs_close(struct rpc_xprt *xprt);
static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
		struct socket *sock);

/*
 * xprtsock tunables
 */

static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE;

static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE;

static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE;


static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT;

static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT;

#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)


#define XS_TCP_LINGER_TO	(15U * HZ)

static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO;

/*
 * We can register our own files under /proc/sys/sunrpc by
 * calling register_sysctl_table() again.  The files in that
 * directory become the union of all files registered there.
 *
 * We simply need to make sure that we don't collide with
 * someone else's file names!
 */


static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE;

static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE;

static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT;

static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT;

static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT;


static struct ctl_table_header *sunrpc_table_header;

/*
 * FIXME: changing the UDP slot table size should also resize the UDP
 *        socket buffers for existing UDP transports
 */

static struct ctl_table xs_tunables_table[] = {
	{
		.procname	= "udp_slot_table_entries",
		.data		= &xprt_udp_slot_table_entries,
		.maxlen		= sizeof(unsigned int),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_minmax,
		.extra1		= &min_slot_table_size,
		.extra2		= &max_slot_table_size
	},
	{
		.procname	= "tcp_slot_table_entries",
		.data		= &xprt_tcp_slot_table_entries,
		.maxlen		= sizeof(unsigned int),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_minmax,
		.extra1		= &min_slot_table_size,
		.extra2		= &max_slot_table_size
	},
	{
		.procname	= "tcp_max_slot_table_entries",
		.data		= &xprt_max_tcp_slot_table_entries,
		.maxlen		= sizeof(unsigned int),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_minmax,
		.extra1		= &min_slot_table_size,
		.extra2		= &max_tcp_slot_table_limit
	},
	{
		.procname	= "min_resvport",
		.data		= &xprt_min_resvport,
		.maxlen		= sizeof(unsigned int),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_minmax,
		.extra1		= &xprt_min_resvport_limit,
		.extra2		= &xprt_max_resvport
	},
	{
		.procname	= "max_resvport",
		.data		= &xprt_max_resvport,
		.maxlen		= sizeof(unsigned int),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_minmax,
		.extra1		= &xprt_min_resvport,
		.extra2		= &xprt_max_resvport_limit
	},
	{
		.procname	= "tcp_fin_timeout",
		.data		= &xs_tcp_fin_timeout,
		.maxlen		= sizeof(xs_tcp_fin_timeout),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_jiffies,
        },
	{ },
};


static struct ctl_table sunrpc_table[] = {
	{
		.procname	= "sunrpc",
		.mode		= 0555,
		.child		= xs_tunables_table
	},
	{ },
};

#endif

/*
 * Wait duration for a reply from the RPC portmapper.
 */

#define XS_BIND_TO		(60U * HZ)

/*
 * Delay if a UDP socket connect error occurs.  This is most likely some
 * kind of resource problem on the local host.
 */

#define XS_UDP_REEST_TO		(2U * HZ)

/*
 * The reestablish timeout allows clients to delay for a bit before attempting
 * to reconnect to a server that just dropped our connection.
 *
 * We implement an exponential backoff when trying to reestablish a TCP
 * transport connection with the server.  Some servers like to drop a TCP
 * connection when they are overworked, so we start with a short timeout and
 * increase over time if the server is down or not responding.
 */

#define XS_TCP_INIT_REEST_TO	(3U * HZ)

/*
 * TCP idle timeout; client drops the transport socket if it is idle
 * for this long.  Note that we also timeout UDP sockets to prevent
 * holding port numbers when there is no RPC traffic.
 */

#define XS_IDLE_DISC_TO		(5U * 60 * HZ)

#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)

# undef  RPC_DEBUG_DATA

# define RPCDBG_FACILITY	RPCDBG_TRANS
#endif

#ifdef RPC_DEBUG_DATA

static void xs_pktdump(char *msg, u32 *packet, unsigned int count) { u8 *buf = (u8 *) packet; int j; dprintk("RPC: %s\n", msg); for (j = 0; j < count && j < 128; j += 4) { if (!(j & 31)) { if (j) dprintk("\n"); dprintk("0x%04x ", j); } dprintk("%02x%02x%02x%02x ", buf[j], buf[j+1], buf[j+2], buf[j+3]); } dprintk("\n"); }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever121100.00%3100.00%
Total121100.00%3100.00%

#else
static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count) { /* NOP */ }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever20100.00%2100.00%
Total20100.00%2100.00%

#endif
static inline struct rpc_xprt *xprt_from_sock(struct sock *sk) { return (struct rpc_xprt *) sk->sk_user_data; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust24100.00%1100.00%
Total24100.00%1100.00%


static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt) { return (struct sockaddr *) &xprt->addr; }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever25100.00%2100.00%
Total25100.00%2100.00%


static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt) { return (struct sockaddr_un *) &xprt->addr; }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever25100.00%1100.00%
Total25100.00%1100.00%


static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt) { return (struct sockaddr_in *) &xprt->addr; }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever25100.00%2100.00%
Total25100.00%2100.00%


static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt) { return (struct sockaddr_in6 *) &xprt->addr; }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever25100.00%1100.00%
Total25100.00%1100.00%


static void xs_format_common_peer_addresses(struct rpc_xprt *xprt) { struct sockaddr *sap = xs_addr(xprt); struct sockaddr_in6 *sin6; struct sockaddr_in *sin; struct sockaddr_un *sun; char buf[128]; switch (sap->sa_family) { case AF_LOCAL: sun = xs_addr_un(xprt); strlcpy(buf, sun->sun_path, sizeof(buf)); xprt->address_strings[RPC_DISPLAY_ADDR] = kstrdup(buf, GFP_KERNEL); break; case AF_INET: (void)rpc_ntop(sap, buf, sizeof(buf)); xprt->address_strings[RPC_DISPLAY_ADDR] = kstrdup(buf, GFP_KERNEL); sin = xs_addr_in(xprt); snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr)); break; case AF_INET6: (void)rpc_ntop(sap, buf, sizeof(buf)); xprt->address_strings[RPC_DISPLAY_ADDR] = kstrdup(buf, GFP_KERNEL); sin6 = xs_addr_in6(xprt); snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr); break; default: BUG(); } xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever22198.22%770.00%
Joe Perches20.89%110.00%
Harvey Harrison10.44%110.00%
Talpey, Thomas10.44%110.00%
Total225100.00%10100.00%


static void xs_format_common_peer_ports(struct rpc_xprt *xprt) { struct sockaddr *sap = xs_addr(xprt); char buf[128]; snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap)); xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap)); xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever89100.00%4100.00%
Total89100.00%4100.00%


static void xs_format_peer_addresses(struct rpc_xprt *xprt, const char *protocol, const char *netid) { xprt->address_strings[RPC_DISPLAY_PROTO] = protocol; xprt->address_strings[RPC_DISPLAY_NETID] = netid; xs_format_common_peer_addresses(xprt); xs_format_common_peer_ports(xprt); }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever49100.00%5100.00%
Total49100.00%5100.00%


static void xs_update_peer_port(struct rpc_xprt *xprt) { kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]); kfree(xprt->address_strings[RPC_DISPLAY_PORT]); xs_format_common_peer_ports(xprt); }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever3083.33%480.00%
Talpey, Thomas616.67%120.00%
Total36100.00%5100.00%


static void xs_free_peer_addresses(struct rpc_xprt *xprt) { unsigned int i; for (i = 0; i < RPC_DISPLAY_MAX; i++) switch (i) { case RPC_DISPLAY_PROTO: case RPC_DISPLAY_NETID: continue; default: kfree(xprt->address_strings[i]); } }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever52100.00%2100.00%
Total52100.00%2100.00%

#define XS_SENDMSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
static int xs_send_kvec(struct socket *sock, struct sockaddr *addr, int addrlen, struct kvec *vec, unsigned int base, int more) { struct msghdr msg = { .msg_name = addr, .msg_namelen = addrlen, .msg_flags = XS_SENDMSG_FLAGS | (more ? MSG_MORE : 0), }; struct kvec iov = { .iov_base = vec->iov_base + base, .iov_len = vec->iov_len - base, }; if (iov.iov_len != 0) return kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len); return kernel_sendmsg(sock, &msg, NULL, 0, 0); }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever8971.20%266.67%
Trond Myklebust3628.80%133.33%
Total125100.00%3100.00%


static int xs_send_pagedata(struct socket *sock, struct xdr_buf *xdr, unsigned int base, int more, bool zerocopy, int *sent_p) { ssize_t (*do_sendpage)(struct socket *sock, struct page *page, int offset, size_t size, int flags); struct page **ppage; unsigned int remainder; int err; remainder = xdr->page_len - base; base += xdr->page_base; ppage = xdr->pages + (base >> PAGE_SHIFT); base &= ~PAGE_MASK; do_sendpage = sock->ops->sendpage; if (!zerocopy) do_sendpage = sock_no_sendpage; for(;;) { unsigned int len = min_t(unsigned int, PAGE_SIZE - base, remainder); int flags = XS_SENDMSG_FLAGS; remainder -= len; if (more) flags |= MSG_MORE; if (remainder != 0) flags |= MSG_SENDPAGE_NOTLAST | MSG_MORE; err = do_sendpage(sock, *ppage, base, len, flags); if (remainder == 0 || err != len) break; *sent_p += err; ppage++; base = 0; } if (err > 0) { *sent_p += err; err = 0; } return err; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust17075.56%360.00%
Chuck Lever4017.78%120.00%
Jason Baron156.67%120.00%
Total225100.00%5100.00%

/** * xs_sendpages - write pages directly to a socket * @sock: socket to send on * @addr: UDP only -- address of destination * @addrlen: UDP only -- length of destination address * @xdr: buffer containing this request * @base: starting position in the buffer * @zerocopy: true if it is safe to use sendpage() * @sent_p: return the total number of bytes successfully queued for sending * */
static int xs_sendpages(struct socket *sock, struct sockaddr *addr, int addrlen, struct xdr_buf *xdr, unsigned int base, bool zerocopy, int *sent_p) { unsigned int remainder = xdr->len - base; int err = 0; int sent = 0; if (unlikely(!sock)) return -ENOTSOCK; if (base != 0) { addr = NULL; addrlen = 0; } if (base < xdr->head[0].iov_len || addr != NULL) { unsigned int len = xdr->head[0].iov_len - base; remainder -= len; err = xs_send_kvec(sock, addr, addrlen, &xdr->head[0], base, remainder != 0); if (remainder == 0 || err != len) goto out; *sent_p += err; base = 0; } else base -= xdr->head[0].iov_len; if (base < xdr->page_len) { unsigned int len = xdr->page_len - base; remainder -= len; err = xs_send_pagedata(sock, xdr, base, remainder != 0, zerocopy, &sent); *sent_p += sent; if (remainder == 0 || sent != len) goto out; base = 0; } else base -= xdr->page_len; if (base >= xdr->tail[0].iov_len) return 0; err = xs_send_kvec(sock, NULL, 0, &xdr->tail[0], base, 0); out: if (err > 0) { *sent_p += err; err = 0; } return err; }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever17254.78%342.86%
Trond Myklebust11335.99%342.86%
Jason Baron299.24%114.29%
Total314100.00%7100.00%


static void xs_nospace_callback(struct rpc_task *task) { struct sock_xprt *transport = container_of(task->tk_rqstp->rq_xprt, struct sock_xprt, xprt); transport->inet->sk_write_pending--; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust37100.00%1100.00%
Total37100.00%1100.00%

/** * xs_nospace - place task on wait queue if transmit was incomplete * @task: task to put to sleep * */
static int xs_nospace(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); struct sock *sk = transport->inet; int ret = -EAGAIN; dprintk("RPC: %5u xmit incomplete (%u left of %u)\n", task->tk_pid, req->rq_slen - req->rq_bytes_sent, req->rq_slen); /* Protect against races with write_space */ spin_lock_bh(&xprt->transport_lock); /* Don't race with disconnect */ if (xprt_connected(xprt)) { /* wait for more buffer space */ sk->sk_write_pending++; xprt_wait_for_buffer_space(task, xs_nospace_callback); } else ret = -ENOTCONN; spin_unlock_bh(&xprt->transport_lock); /* Race breaker in case memory is freed before above code is called */ if (ret == -EAGAIN) { struct socket_wq *wq; rcu_read_lock(); wq = rcu_dereference(sk->sk_wq); set_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags); rcu_read_unlock(); sk->sk_write_space(sk); } return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever8950.57%440.00%
Trond Myklebust4827.27%550.00%
David Vrabel3922.16%110.00%
Total176100.00%10100.00%

/* * Construct a stream transport record marker in @buf. */
static inline void xs_encode_stream_record_marker(struct xdr_buf *buf) { u32 reclen = buf->len - sizeof(rpc_fraghdr); rpc_fraghdr *base = buf->head[0].iov_base; *base = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | reclen); }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever47100.00%1100.00%
Total47100.00%1100.00%

/** * xs_local_send_request - write an RPC request to an AF_LOCAL socket * @task: RPC task that manages the state of an RPC request * * Return values: * 0: The request has been sent * EAGAIN: The socket was blocked, please call again later to * complete the request * ENOTCONN: Caller needs to invoke connect logic then call again * other: Some other error occured, the request was not sent */
static int xs_local_send_request(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); struct xdr_buf *xdr = &req->rq_snd_buf; int status; int sent = 0; xs_encode_stream_record_marker(&req->rq_snd_buf); xs_pktdump("packet data:", req->rq_svec->iov_base, req->rq_svec->iov_len); status = xs_sendpages(transport->sock, NULL, 0, xdr, req->rq_bytes_sent, true, &sent); dprintk("RPC: %s(%u) = %d\n", __func__, xdr->len - req->rq_bytes_sent, status); if (status == -EAGAIN && sock_writeable(transport->inet)) status = -ENOBUFS; if (likely(sent > 0) || status == 0) { req->rq_bytes_sent += sent; req->rq_xmit_bytes_sent += sent; if (likely(req->rq_bytes_sent >= req->rq_slen)) { req->rq_bytes_sent = 0; return 0; } status = -EAGAIN; } switch (status) { case -ENOBUFS: break; case -EAGAIN: status = xs_nospace(task); break; default: dprintk("RPC: sendmsg returned unrecognized error %d\n", -status); case -EPIPE: xs_close(xprt); status = -ENOTCONN; } return status; }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever18171.83%637.50%
Trond Myklebust3614.29%850.00%
Neil Brown197.54%16.25%
Jason Baron166.35%16.25%
Total252100.00%16100.00%

/** * xs_udp_send_request - write an RPC request to a UDP socket * @task: address of RPC task that manages the state of an RPC request * * Return values: * 0: The request has been sent * EAGAIN: The socket was blocked, please call again later to * complete the request * ENOTCONN: Caller needs to invoke connect logic then call again * other: Some other error occurred, the request was not sent */
static int xs_udp_send_request(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); struct xdr_buf *xdr = &req->rq_snd_buf; int sent = 0; int status; xs_pktdump("packet data:", req->rq_svec->iov_base, req->rq_svec->iov_len); if (!xprt_bound(xprt)) return -ENOTCONN; status = xs_sendpages(transport->sock, xs_addr(xprt), xprt->addrlen, xdr, req->rq_bytes_sent, true, &sent); dprintk("RPC: xs_udp_send_request(%u) = %d\n", xdr->len - req->rq_bytes_sent, status); /* firewall is blocking us, don't return -EAGAIN or we end up looping */ if (status == -EPERM) goto process_status; if (status == -EAGAIN && sock_writeable(transport->inet)) status = -ENOBUFS; if (sent > 0 || status == 0) { req->rq_xmit_bytes_sent += sent; if (sent >= req->rq_slen) return 0; /* Still some bytes left; set up for a retry later. */ status = -EAGAIN; } process_status: switch (status) { case -ENOTSOCK: status = -ENOTCONN; /* Should we call xs_close() here? */ break; case -EAGAIN: status = xs_nospace(task); break; case -ENETUNREACH: case -ENOBUFS: case -EPIPE: case -ECONNREFUSED: case -EPERM: /* When the server has died, an ICMP port unreachable message * prompts ECONNREFUSED. */ break; default: dprintk("RPC: sendmsg returned unrecognized error %d\n", -status); } return status; }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever17967.55%112.50%
Trond Myklebust3513.21%450.00%
Jason Baron3212.08%225.00%
Neil Brown197.17%112.50%
Total265100.00%8100.00%

/** * xs_tcp_send_request - write an RPC request to a TCP socket * @task: address of RPC task that manages the state of an RPC request * * Return values: * 0: The request has been sent * EAGAIN: The socket was blocked, please call again later to * complete the request * ENOTCONN: Caller needs to invoke connect logic then call again * other: Some other error occurred, the request was not sent * * XXX: In the case of soft timeouts, should we eventually give up * if sendmsg is not able to make progress? */
static int xs_tcp_send_request(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); struct xdr_buf *xdr = &req->rq_snd_buf; bool zerocopy = true; bool vm_wait = false; int status; int sent; xs_encode_stream_record_marker(&req->rq_snd_buf); xs_pktdump("packet data:", req->rq_svec->iov_base, req->rq_svec->iov_len); /* Don't use zero copy if this is a resend. If the RPC call * completes while the socket holds a reference to the pages, * then we may end up resending corrupted data. */ if (task->tk_flags & RPC_TASK_SENT) zerocopy = false; if (test_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state)) xs_tcp_set_socket_timeouts(xprt, transport->sock); /* Continue transmitting the packet/record. We must be careful * to cope with writespace callbacks arriving _after_ we have * called sendmsg(). */ while (1) { sent = 0; status = xs_sendpages(transport->sock, NULL, 0, xdr, req->rq_bytes_sent, zerocopy, &sent); dprintk("RPC: xs_tcp_send_request(%u) = %d\n", xdr->len - req->rq_bytes_sent, status); /* If we've sent the entire packet, immediately * reset the count of bytes sent. */ req->rq_bytes_sent += sent; req->rq_xmit_bytes_sent += sent; if (likely(req->rq_bytes_sent >= req->rq_slen)) { req->rq_bytes_sent = 0; return 0; } WARN_ON_ONCE(sent == 0 && status == 0); if (status == -EAGAIN ) { /* * Return EAGAIN if we're sure we're hitting the * socket send buffer limits. */ if (test_bit(SOCK_NOSPACE, &transport->sock->flags)) break; /* * Did we hit a memory allocation failure? */ if (sent == 0) { status = -