cregit-Linux how code gets into the kernel

Release 4.11 net/smc/af_smc.c

Directory: net/smc
/*
 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
 *  applies to SOCK_STREAM sockets only
 *  offers an alternative communication option for TCP-protocol sockets
 *  applicable with RoCE-cards only
 *
 *  Initial restrictions:
 *    - non-blocking connect postponed
 *    - IPv6 support postponed
 *    - support for alternate links postponed
 *    - partial support for non-blocking sockets only
 *    - support for urgent data postponed
 *
 *  Copyright IBM Corp. 2016
 *
 *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
 *              based on prototype from Frank Blaschka
 */


#define KMSG_COMPONENT "smc"

#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/socket.h>
#include <linux/inetdevice.h>
#include <linux/workqueue.h>
#include <linux/in.h>
#include <linux/sched/signal.h>

#include <net/sock.h>
#include <net/tcp.h>
#include <net/smc.h>

#include "smc.h"
#include "smc_clc.h"
#include "smc_llc.h"
#include "smc_cdc.h"
#include "smc_core.h"
#include "smc_ib.h"
#include "smc_pnet.h"
#include "smc_tx.h"
#include "smc_rx.h"
#include "smc_close.h"

static DEFINE_MUTEX(smc_create_lgr_pending);	/* serialize link group
                                                 * creation
                                                 */


struct smc_lgr_list smc_lgr_list = {		/* established link groups */
	.lock = __SPIN_LOCK_UNLOCKED(smc_lgr_list.lock),
	.list = LIST_HEAD_INIT(smc_lgr_list.list),
};

static void smc_tcp_listen_work(struct work_struct *);


static void smc_set_keepalive(struct sock *sk, int val) { struct smc_sock *smc = smc_sk(sk); smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val); }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun43100.00%1100.00%
Total43100.00%1100.00%

static struct smc_hashinfo smc_v4_hashinfo = { .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock), };
int smc_hash_sk(struct sock *sk) { struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; struct hlist_head *head; head = &h->ht; write_lock_bh(&h->lock); sk_add_node(sk, head); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); write_unlock_bh(&h->lock); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun75100.00%1100.00%
Total75100.00%1100.00%

EXPORT_SYMBOL_GPL(smc_hash_sk);
void smc_unhash_sk(struct sock *sk) { struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; write_lock_bh(&h->lock); if (sk_del_node_init(sk)) sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); write_unlock_bh(&h->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun61100.00%1100.00%
Total61100.00%1100.00%

EXPORT_SYMBOL_GPL(smc_unhash_sk); struct proto smc_proto = { .name = "SMC", .owner = THIS_MODULE, .keepalive = smc_set_keepalive, .hash = smc_hash_sk, .unhash = smc_unhash_sk, .obj_size = sizeof(struct smc_sock), .h.smc_hash = &smc_v4_hashinfo, .slab_flags = SLAB_DESTROY_BY_RCU, }; EXPORT_SYMBOL_GPL(smc_proto);
static int smc_release(struct socket *sock) { struct sock *sk = sock->sk; struct smc_sock *smc; int rc = 0; if (!sk) goto out; smc = smc_sk(sk); sock_hold(sk); if (sk->sk_state == SMC_LISTEN) /* smc_close_non_accepted() is called and acquires * sock lock for child sockets again */ lock_sock_nested(sk, SINGLE_DEPTH_NESTING); else lock_sock(sk); if (smc->use_fallback) { sk->sk_state = SMC_CLOSED; sk->sk_state_change(sk); } else { rc = smc_close_active(smc); sock_set_flag(sk, SOCK_DEAD); sk->sk_shutdown |= SHUTDOWN_MASK; } if (smc->clcsock) { sock_release(smc->clcsock); smc->clcsock = NULL; } /* detach socket */ sock_orphan(sk); sock->sk = NULL; if (smc->use_fallback) { schedule_delayed_work(&smc->sock_put_work, TCP_TIMEWAIT_LEN); } else if (sk->sk_state == SMC_CLOSED) { smc_conn_free(&smc->conn); schedule_delayed_work(&smc->sock_put_work, SMC_CLOSE_SOCK_PUT_DELAY); } sk->sk_prot->unhash(sk); release_sock(sk); sock_put(sk); out: return rc; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun220100.00%3100.00%
Total220100.00%3100.00%


static void smc_destruct(struct sock *sk) { if (sk->sk_state != SMC_CLOSED) return; if (!sock_flag(sk, SOCK_DEAD)) return; sk_refcnt_debug_dec(sk); }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun36100.00%1100.00%
Total36100.00%1100.00%


static struct sock *smc_sock_alloc(struct net *net, struct socket *sock) { struct smc_sock *smc; struct sock *sk; sk = sk_alloc(net, PF_SMC, GFP_KERNEL, &smc_proto, 0); if (!sk) return NULL; sock_init_data(sock, sk); /* sets sk_refcnt to 1 */ sk->sk_state = SMC_INIT; sk->sk_destruct = smc_destruct; sk->sk_protocol = SMCPROTO_SMC; smc = smc_sk(sk); INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work); INIT_LIST_HEAD(&smc->accept_q); spin_lock_init(&smc->accept_q_lock); INIT_DELAYED_WORK(&smc->sock_put_work, smc_close_sock_put_work); sk->sk_prot->hash(sk); sk_refcnt_debug_inc(sk); return sk; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun138100.00%4100.00%
Total138100.00%4100.00%


static int smc_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) { struct sockaddr_in *addr = (struct sockaddr_in *)uaddr; struct sock *sk = sock->sk; struct smc_sock *smc; int rc; smc = smc_sk(sk); /* replicate tests from inet_bind(), to be safe wrt. future changes */ rc = -EINVAL; if (addr_len < sizeof(struct sockaddr_in)) goto out; rc = -EAFNOSUPPORT; /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */ if ((addr->sin_family != AF_INET) && ((addr->sin_family != AF_UNSPEC) || (addr->sin_addr.s_addr != htonl(INADDR_ANY)))) goto out; lock_sock(sk); /* Check if socket is already active */ rc = -EINVAL; if (sk->sk_state != SMC_INIT) goto out_rel; smc->clcsock->sk->sk_reuse = sk->sk_reuse; rc = kernel_bind(smc->clcsock, uaddr, addr_len); out_rel: release_sock(sk); out: return rc; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun175100.00%1100.00%
Total175100.00%1100.00%


static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk, unsigned long mask) { /* options we don't get control via setsockopt for */ nsk->sk_type = osk->sk_type; nsk->sk_sndbuf = osk->sk_sndbuf; nsk->sk_rcvbuf = osk->sk_rcvbuf; nsk->sk_sndtimeo = osk->sk_sndtimeo; nsk->sk_rcvtimeo = osk->sk_rcvtimeo; nsk->sk_mark = osk->sk_mark; nsk->sk_priority = osk->sk_priority; nsk->sk_rcvlowat = osk->sk_rcvlowat; nsk->sk_bound_dev_if = osk->sk_bound_dev_if; nsk->sk_err = osk->sk_err; nsk->sk_flags &= ~mask; nsk->sk_flags |= osk->sk_flags & mask; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun118100.00%1100.00%
Total118100.00%1100.00%

#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \ (1UL << SOCK_KEEPOPEN) | \ (1UL << SOCK_LINGER) | \ (1UL << SOCK_BROADCAST) | \ (1UL << SOCK_TIMESTAMP) | \ (1UL << SOCK_DBG) | \ (1UL << SOCK_RCVTSTAMP) | \ (1UL << SOCK_RCVTSTAMPNS) | \ (1UL << SOCK_LOCALROUTE) | \ (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \ (1UL << SOCK_RXQ_OVFL) | \ (1UL << SOCK_WIFI_STATUS) | \ (1UL << SOCK_NOFCS) | \ (1UL << SOCK_FILTER_LOCKED)) /* copy only relevant settings and flags of SOL_SOCKET level from smc to * clc socket (since smc is not called for these options from net/core) */
static void smc_copy_sock_settings_to_clc(struct smc_sock *smc) { smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC); }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun27100.00%1100.00%
Total27100.00%1100.00%

#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \ (1UL << SOCK_KEEPOPEN) | \ (1UL << SOCK_LINGER) | \ (1UL << SOCK_DBG)) /* copy only settings and flags relevant for smc from clc to smc socket */
static void smc_copy_sock_settings_to_smc(struct smc_sock *smc) { smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC); }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun27100.00%1100.00%
Total27100.00%1100.00%

/* determine subnet and mask of internal TCP socket */
int smc_netinfo_by_tcpsk(struct socket *clcsock, __be32 *subnet, u8 *prefix_len) { struct dst_entry *dst = sk_dst_get(clcsock->sk); struct sockaddr_in addr; int rc = -ENOENT; int len; if (!dst) { rc = -ENOTCONN; goto out; } if (!dst->dev) { rc = -ENODEV; goto out_rel; } /* get address to which the internal TCP socket is bound */ kernel_getsockname(clcsock, (struct sockaddr *)&addr, &len); /* analyze IPv4 specific data of net_device belonging to TCP socket */ for_ifa(dst->dev->ip_ptr) { if (ifa->ifa_address != addr.sin_addr.s_addr) continue; *prefix_len = inet_mask_len(ifa->ifa_mask); *subnet = ifa->ifa_address & ifa->ifa_mask; rc = 0; break; } endfor_ifa(dst->dev->ip_ptr); out_rel: dst_release(dst); out: return rc; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun159100.00%2100.00%
Total159100.00%2100.00%


static int smc_clnt_conf_first_link(struct smc_sock *smc, union ib_gid *gid) { struct smc_link_group *lgr = smc->conn.lgr; struct smc_link *link; int rest; int rc; link = &lgr->lnk[SMC_SINGLE_LINK]; /* receive CONFIRM LINK request from server over RoCE fabric */ rest = wait_for_completion_interruptible_timeout( &link->llc_confirm, SMC_LLC_WAIT_FIRST_TIME); if (rest <= 0) { struct smc_clc_msg_decline dclc; rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), SMC_CLC_DECLINE); return rc; } rc = smc_ib_modify_qp_rts(link); if (rc) return SMC_CLC_DECL_INTERR; smc_wr_remember_qp_attr(link); /* send CONFIRM LINK response over RoCE fabric */ rc = smc_llc_send_confirm_link(link, link->smcibdev->mac[link->ibport - 1], gid, SMC_LLC_RESP); if (rc < 0) return SMC_CLC_DECL_TCL; return rc; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun149100.00%1100.00%
Total149100.00%1100.00%


static void smc_conn_save_peer_info(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *clc) { smc->conn.peer_conn_idx = clc->conn_idx; smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token); smc->conn.peer_rmbe_size = smc_uncompress_bufsize(clc->rmbe_size); atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun70100.00%3100.00%
Total70100.00%3100.00%


static void smc_link_save_peer_info(struct smc_link *link, struct smc_clc_msg_accept_confirm *clc) { link->peer_qpn = ntoh24(clc->qpn); memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE); memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac)); link->peer_psn = ntoh24(clc->psn); link->peer_mtu = clc->qp_mtu; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun81100.00%1100.00%
Total81100.00%1100.00%

/* setup for RDMA connection of client */
static int smc_connect_rdma(struct smc_sock *smc) { struct sockaddr_in *inaddr = (struct sockaddr_in *)smc->addr; struct smc_clc_msg_accept_confirm aclc; int local_contact = SMC_FIRST_CONTACT; struct smc_ib_device *smcibdev; struct smc_link *link; u8 srv_first_contact; int reason_code = 0; int rc = 0; u8 ibport; /* IPSec connections opt out of SMC-R optimizations */ if (using_ipsec(smc)) { reason_code = SMC_CLC_DECL_IPSEC; goto decline_rdma; } /* PNET table look up: search active ib_device and port * within same PNETID that also contains the ethernet device * used for the internal TCP socket */ smc_pnet_find_roce_resource(smc->clcsock->sk, &smcibdev, &ibport); if (!smcibdev) { reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */ goto decline_rdma; } /* do inband token exchange */ reason_code = smc_clc_send_proposal(smc, smcibdev, ibport); if (reason_code < 0) { rc = reason_code; goto out_err; } if (reason_code > 0) /* configuration error */ goto decline_rdma; /* receive SMC Accept CLC message */ reason_code = smc_clc_wait_msg(smc, &aclc, sizeof(aclc), SMC_CLC_ACCEPT); if (reason_code < 0) { rc = reason_code; goto out_err; } if (reason_code > 0) goto decline_rdma; srv_first_contact = aclc.hdr.flag; mutex_lock(&smc_create_lgr_pending); local_contact = smc_conn_create(smc, inaddr->sin_addr.s_addr, smcibdev, ibport, &aclc.lcl, srv_first_contact); if (local_contact < 0) { rc = local_contact; if (rc == -ENOMEM) reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/ else if (rc == -ENOLINK) reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */ goto decline_rdma_unlock; } link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK]; smc_conn_save_peer_info(smc, &aclc); rc = smc_sndbuf_create(smc); if (rc) { reason_code = SMC_CLC_DECL_MEM; goto decline_rdma_unlock; } rc = smc_rmb_create(smc); if (rc) { reason_code = SMC_CLC_DECL_MEM; goto decline_rdma_unlock; } if (local_contact == SMC_FIRST_CONTACT) smc_link_save_peer_info(link, &aclc); rc = smc_rmb_rtoken_handling(&smc->conn, &aclc); if (rc) { reason_code = SMC_CLC_DECL_INTERR; goto decline_rdma_unlock; } if (local_contact == SMC_FIRST_CONTACT) { rc = smc_ib_ready_link(link); if (rc) { reason_code = SMC_CLC_DECL_INTERR; goto decline_rdma_unlock; } } rc = smc_clc_send_confirm(smc); if (rc) goto out_err_unlock; if (local_contact == SMC_FIRST_CONTACT) { /* QP confirmation over RoCE fabric */ reason_code = smc_clnt_conf_first_link( smc, &smcibdev->gid[ibport - 1]); if (reason_code < 0) { rc = reason_code; goto out_err_unlock; } if (reason_code > 0) goto decline_rdma_unlock; } mutex_unlock(&smc_create_lgr_pending); smc_tx_init(smc); smc_rx_init(smc); out_connected: smc_copy_sock_settings_to_clc(smc); if (smc->sk.sk_state == SMC_INIT) smc->sk.sk_state = SMC_ACTIVE; return rc ? rc : local_contact; decline_rdma_unlock: mutex_unlock(&smc_create_lgr_pending); smc_conn_free(&smc->conn); decline_rdma: /* RDMA setup failed, switch back to TCP */ smc->use_fallback = true; if (reason_code && (reason_code != SMC_CLC_DECL_REPLY)) { rc = smc_clc_send_decline(smc, reason_code, 0); if (rc < sizeof(struct smc_clc_msg_decline)) goto out_err; } goto out_connected; out_err_unlock: mutex_unlock(&smc_create_lgr_pending); smc_conn_free(&smc->conn); out_err: return rc; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun592100.00%9100.00%
Total592100.00%9100.00%


static int smc_connect(struct socket *sock, struct sockaddr *addr, int alen, int flags) { struct sock *sk = sock->sk; struct smc_sock *smc; int rc = -EINVAL; smc = smc_sk(sk); /* separate smc parameter checking to be safe */ if (alen < sizeof(addr->sa_family)) goto out_err; if (addr->sa_family != AF_INET) goto out_err; smc->addr = addr; /* needed for nonblocking connect */ lock_sock(sk); switch (sk->sk_state) { default: goto out; case SMC_ACTIVE: rc = -EISCONN; goto out; case SMC_INIT: rc = 0; break; } smc_copy_sock_settings_to_clc(smc); rc = kernel_connect(smc->clcsock, addr, alen, flags); if (rc) goto out; /* setup RDMA connection */ rc = smc_connect_rdma(smc); if (rc < 0) goto out; else rc = 0; /* success cases including fallback */ out: release_sock(sk); out_err: return rc; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun180100.00%2100.00%
Total180100.00%2100.00%


static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc) { struct sock *sk = &lsmc->sk; struct socket *new_clcsock; struct sock *new_sk; int rc; release_sock(&lsmc->sk); new_sk = smc_sock_alloc(sock_net(sk), NULL); if (!new_sk) { rc = -ENOMEM; lsmc->sk.sk_err = ENOMEM; *new_smc = NULL; lock_sock(&lsmc->sk); goto out; } *new_smc = smc_sk(new_sk); rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0); lock_sock(&lsmc->sk); if (rc < 0) { lsmc->sk.sk_err = -rc; new_sk->sk_state = SMC_CLOSED; sock_set_flag(new_sk, SOCK_DEAD); sk->sk_prot->unhash(new_sk); sock_put(new_sk); *new_smc = NULL; goto out; } if (lsmc->sk.sk_state == SMC_CLOSED) { if (new_clcsock) sock_release(new_clcsock); new_sk->sk_state = SMC_CLOSED; sock_set_flag(new_sk, SOCK_DEAD); sk->sk_prot->unhash(new_sk); sock_put(new_sk); *new_smc = NULL; goto out; } (*new_smc)->clcsock = new_clcsock; out: return rc; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun248100.00%2100.00%
Total248100.00%2100.00%

/* add a just created sock to the accept queue of the listen sock as * candidate for a following socket accept call from user space */
static void smc_accept_enqueue(struct sock *parent, struct sock *sk) { struct smc_sock *par = smc_sk(parent); sock_hold(sk); spin_lock(&par->accept_q_lock); list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q); spin_unlock(&par->accept_q_lock); sk_acceptq_added(parent); }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun68100.00%1100.00%
Total68100.00%1100.00%

/* remove a socket from the accept queue of its parental listening socket */
static void smc_accept_unlink(struct sock *sk) { struct smc_sock *par = smc_sk(sk)->listen_smc; spin_lock(&par->accept_q_lock); list_del_init(&smc_sk(sk)->accept_q); spin_unlock(&par->accept_q_lock); sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk); sock_put(sk); }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun68100.00%1100.00%
Total68100.00%1100.00%

/* remove a sock from the accept queue to bind it to a new socket created * for a socket accept call from user space */
struct sock *smc_accept_dequeue(struct sock *parent, struct socket *new_sock) { struct smc_sock *isk, *n; struct sock *new_sk; list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) { new_sk = (struct sock *)isk; smc_accept_unlink(new_sk); if (new_sk->sk_state == SMC_CLOSED) { /* tbd in follow-on patch: close this sock */ continue; } if (new_sock) sock_graft(new_sk, new_sock); return new_sk; } return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun85100.00%1100.00%
Total85100.00%1100.00%

/* clean up for a created but never accepted sock */
void smc_close_non_accepted(struct sock *sk) { struct smc_sock *smc = smc_sk(sk); sock_hold(sk); lock_sock(sk); if (!sk->sk_lingertime) /* wait for peer closing */ sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT; if (!smc->use_fallback) smc_close_active(smc); if (smc->clcsock) { struct socket *tcp; tcp = smc->clcsock; smc->clcsock = NULL; sock_release(tcp); } sock_set_flag(sk, SOCK_DEAD); sk->sk_shutdown |= SHUTDOWN_MASK; if (smc->use_fallback) { schedule_delayed_work(&smc->sock_put_work, TCP_TIMEWAIT_LEN); } else { smc_conn_free(&smc->conn); schedule_delayed_work(&smc->sock_put_work, SMC_CLOSE_SOCK_PUT_DELAY); } release_sock(sk); sock_put(sk); }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun148100.00%2100.00%
Total148100.00%2100.00%


static int smc_serv_conf_first_link(struct smc_sock *smc) { struct smc_link_group *lgr = smc->conn.lgr; struct smc_link *link; int rest; int rc; link = &lgr->lnk[SMC_SINGLE_LINK]; /* send CONFIRM LINK request to client over the RoCE fabric */ rc = smc_llc_send_confirm_link(link, link->smcibdev->mac[link->ibport - 1], &link->smcibdev->gid[link->ibport - 1], SMC_LLC_REQ); if (rc < 0) return SMC_CLC_DECL_TCL; /* receive CONFIRM LINK response from client over the RoCE fabric */ rest = wait_for_completion_interruptible_timeout( &link->llc_confirm_resp, SMC_LLC_WAIT_FIRST_TIME); if (rest <= 0) { struct smc_clc_msg_decline dclc; rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), SMC_CLC_DECLINE); } return rc; }

Contributors

PersonTokensPropCommitsCommitProp
Ursula Braun134100.00%1100.00%
Total134100.00%1100.00%