cregit-Linux how code gets into the kernel

Release 4.11 crypto/algif_skcipher.c

Directory: crypto
/*
 * algif_skcipher: User-space interface for skcipher algorithms
 *
 * This file provides the user-space API for symmetric key ciphers.
 *
 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 */

#include <crypto/scatterwalk.h>
#include <crypto/skcipher.h>
#include <crypto/if_alg.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/net.h>
#include <net/sock.h>


struct skcipher_sg_list {
	
struct list_head list;

	
int cur;

	
struct scatterlist sg[0];
};


struct skcipher_tfm {
	
struct crypto_skcipher *skcipher;
	
bool has_key;
};


struct skcipher_ctx {
	
struct list_head tsgl;
	
struct af_alg_sgl rsgl;

	
void *iv;

	
struct af_alg_completion completion;

	
atomic_t inflight;
	
size_t used;

	
unsigned int len;
	
bool more;
	
bool merge;
	
bool enc;

	
struct skcipher_request req;
};


struct skcipher_async_rsgl {
	
struct af_alg_sgl sgl;
	
struct list_head list;
};


struct skcipher_async_req {
	
struct kiocb *iocb;
	
struct skcipher_async_rsgl first_sgl;
	
struct list_head list;
	
struct scatterlist *tsg;
	
atomic_t *inflight;
	
struct skcipher_request req;
};


#define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_sg_list)) / \
                      sizeof(struct scatterlist) - 1)


static void skcipher_free_async_sgls(struct skcipher_async_req *sreq) { struct skcipher_async_rsgl *rsgl, *tmp; struct scatterlist *sgl; struct scatterlist *sg; int i, n; list_for_each_entry_safe(rsgl, tmp, &sreq->list, list) { af_alg_free_sg(&rsgl->sgl); if (rsgl != &sreq->first_sgl) kfree(rsgl); } sgl = sreq->tsg; n = sg_nents(sgl); for_each_sg(sgl, sg, n, i) put_page(sg_page(sg)); kfree(sreq->tsg); }

Contributors

PersonTokensPropCommitsCommitProp
Tadeusz Struk106100.00%1100.00%
Total106100.00%1100.00%


static void skcipher_async_cb(struct crypto_async_request *req, int err) { struct skcipher_async_req *sreq = req->data; struct kiocb *iocb = sreq->iocb; atomic_dec(sreq->inflight); skcipher_free_async_sgls(sreq); kzfree(sreq); iocb->ki_complete(iocb, err, err); }

Contributors

PersonTokensPropCommitsCommitProp
Tadeusz Struk5286.67%133.33%
Herbert Xu58.33%133.33%
Al Viro35.00%133.33%
Total60100.00%3100.00%


static inline int skcipher_sndbuf(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) - ctx->used, 0); }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu56100.00%2100.00%
Total56100.00%2100.00%


static inline bool skcipher_writable(struct sock *sk) { return PAGE_SIZE <= skcipher_sndbuf(sk); }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu20100.00%2100.00%
Total20100.00%2100.00%


static int skcipher_alloc_sgl(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; struct skcipher_sg_list *sgl; struct scatterlist *sg = NULL; sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); if (!list_empty(&ctx->tsgl)) sg = sgl->sg; if (!sg || sgl->cur >= MAX_SGL_ENTS) { sgl = sock_kmalloc(sk, sizeof(*sgl) + sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1), GFP_KERNEL); if (!sgl) return -ENOMEM; sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); sgl->cur = 0; if (sg) sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); list_add_tail(&sgl->list, &ctx->tsgl); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu17799.44%150.00%
Dan J Williams10.56%150.00%
Total178100.00%2100.00%


static void skcipher_pull_sgl(struct sock *sk, size_t used, int put) { struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; struct skcipher_sg_list *sgl; struct scatterlist *sg; int i; while (!list_empty(&ctx->tsgl)) { sgl = list_first_entry(&ctx->tsgl, struct skcipher_sg_list, list); sg = sgl->sg; for (i = 0; i < sgl->cur; i++) { size_t plen = min_t(size_t, used, sg[i].length); if (!sg_page(sg + i)) continue; sg[i].length -= plen; sg[i].offset += plen; used -= plen; ctx->used -= plen; if (sg[i].length) return; if (put) put_page(sg_page(sg + i)); sg_assign_page(sg + i, NULL); } list_del(&sgl->list); sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1)); } if (!ctx->used) ctx->merge = 0; }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu22995.82%133.33%
Tadeusz Struk72.93%133.33%
Corentin Labbe31.26%133.33%
Total239100.00%3100.00%


static void skcipher_free_sgl(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; skcipher_pull_sgl(sk, ctx->used, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu3995.12%150.00%
Tadeusz Struk24.88%150.00%
Total41100.00%2100.00%


static int skcipher_wait_for_wmem(struct sock *sk, unsigned flags) { DEFINE_WAIT_FUNC(wait, woken_wake_function); int err = -ERESTARTSYS; long timeout; if (flags & MSG_DONTWAIT) return -EAGAIN; sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); add_wait_queue(sk_sleep(sk), &wait); for (;;) { if (signal_pending(current)) break; timeout = MAX_SCHEDULE_TIMEOUT; if (sk_wait_event(sk, &timeout, skcipher_writable(sk), &wait)) { err = 0; break; } } remove_wait_queue(sk_sleep(sk), &wait); return err; }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu9380.17%133.33%
Américo Wang2118.10%133.33%
Eric Dumazet21.72%133.33%
Total116100.00%3100.00%


static void skcipher_wmem_wakeup(struct sock *sk) { struct socket_wq *wq; if (!skcipher_writable(sk)) return; rcu_read_lock(); wq = rcu_dereference(sk->sk_wq); if (skwq_has_sleeper(wq)) wake_up_interruptible_sync_poll(&wq->wait, POLLIN | POLLRDNORM | POLLRDBAND); sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); rcu_read_unlock(); }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu70100.00%2100.00%
Total70100.00%2100.00%


static int skcipher_wait_for_data(struct sock *sk, unsigned flags) { DEFINE_WAIT_FUNC(wait, woken_wake_function); struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; long timeout; int err = -ERESTARTSYS; if (flags & MSG_DONTWAIT) { return -EAGAIN; } sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); add_wait_queue(sk_sleep(sk), &wait); for (;;) { if (signal_pending(current)) break; timeout = MAX_SCHEDULE_TIMEOUT; if (sk_wait_event(sk, &timeout, ctx->used, &wait)) { err = 0; break; } } remove_wait_queue(sk_sleep(sk), &wait); sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); return err; }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu11781.82%133.33%
Américo Wang2215.38%133.33%
Eric Dumazet42.80%133.33%
Total143100.00%3100.00%


static void skcipher_data_wakeup(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; struct socket_wq *wq; if (!ctx->used) return; rcu_read_lock(); wq = rcu_dereference(sk->sk_wq); if (skwq_has_sleeper(wq)) wake_up_interruptible_sync_poll(&wq->wait, POLLOUT | POLLRDNORM | POLLRDBAND); sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); rcu_read_unlock(); }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu88100.00%2100.00%
Total88100.00%2100.00%


static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) { struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); struct skcipher_ctx *ctx = ask->private; struct skcipher_tfm *skc = pask->private; struct crypto_skcipher *tfm = skc->skcipher; unsigned ivsize = crypto_skcipher_ivsize(tfm); struct skcipher_sg_list *sgl; struct af_alg_control con = {}; long copied = 0; bool enc = 0; bool init = 0; int err; int i; if (msg->msg_controllen) { err = af_alg_cmsg_send(msg, &con); if (err) return err; init = 1; switch (con.op) { case ALG_OP_ENCRYPT: enc = 1; break; case ALG_OP_DECRYPT: enc = 0; break; default: return -EINVAL; } if (con.iv && con.iv->ivlen != ivsize) return -EINVAL; } err = -EINVAL; lock_sock(sk); if (!ctx->more && ctx->used) goto unlock; if (init) { ctx->enc = enc; if (con.iv) memcpy(ctx->iv, con.iv->iv, ivsize); } while (size) { struct scatterlist *sg; unsigned long len = size; size_t plen; if (ctx->merge) { sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); sg = sgl->sg + sgl->cur - 1; len = min_t(unsigned long, len, PAGE_SIZE - sg->offset - sg->length); err = memcpy_from_msg(page_address(sg_page(sg)) + sg->offset + sg->length, msg, len); if (err) goto unlock; sg->length += len; ctx->merge = (sg->offset + sg->length) & (PAGE_SIZE - 1); ctx->used += len; copied += len; size -= len; continue; } if (!skcipher_writable(sk)) { err = skcipher_wait_for_wmem(sk, msg->msg_flags); if (err) goto unlock; } len = min_t(unsigned long, len, skcipher_sndbuf(sk)); err = skcipher_alloc_sgl(sk); if (err) goto unlock; sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); sg = sgl->sg; if (sgl->cur) sg_unmark_end(sg + sgl->cur - 1); do { i = sgl->cur; plen = min_t(size_t, len, PAGE_SIZE); sg_assign_page(sg + i, alloc_page(GFP_KERNEL)); err = -ENOMEM; if (!sg_page(sg + i)) goto unlock; err = memcpy_from_msg(page_address(sg_page(sg + i)), msg, plen); if (err) { __free_page(sg_page(sg + i)); sg_assign_page(sg + i, NULL); goto unlock; } sg[i].length = plen; len -= plen; ctx->used += plen; copied += plen; size -= plen; sgl->cur++; } while (len && sgl->cur < MAX_SGL_ENTS); if (!size) sg_mark_end(sg + sgl->cur - 1); ctx->merge = plen & (PAGE_SIZE - 1); } err = 0; ctx->more = msg->msg_flags & MSG_MORE; unlock: skcipher_data_wakeup(sk); release_sock(sk); return copied ?: err; }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu65094.34%555.56%
Tadeusz Struk253.63%111.11%
Stephan Mueller101.45%111.11%
Al Viro20.29%111.11%
Corentin Labbe20.29%111.11%
Total689100.00%9100.00%


static ssize_t skcipher_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags) { struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; struct skcipher_sg_list *sgl; int err = -EINVAL; if (flags & MSG_SENDPAGE_NOTLAST) flags |= MSG_MORE; lock_sock(sk); if (!ctx->more && ctx->used) goto unlock; if (!size) goto done; if (!skcipher_writable(sk)) { err = skcipher_wait_for_wmem(sk, flags); if (err) goto unlock; } err = skcipher_alloc_sgl(sk); if (err) goto unlock; ctx->merge = 0; sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); if (sgl->cur) sg_unmark_end(sgl->sg + sgl->cur - 1); sg_mark_end(sgl->sg + sgl->cur); get_page(page); sg_set_page(sgl->sg + sgl->cur, page, size, offset); sgl->cur++; ctx->used += size; done: ctx->more = flags & MSG_MORE; unlock: skcipher_data_wakeup(sk); release_sock(sk); return err ?: size; }

Contributors

PersonTokensPropCommitsCommitProp
Herbert Xu21484.25%250.00%
Tadeusz Struk3011.81%125.00%
Shawn Landden103.94%125.00%
Total254100.00%4100.00%


static int skcipher_all_sg_nents(struct skcipher_ctx *ctx) { struct skcipher_sg_list *sgl; struct scatterlist *sg; int nents = 0; list_for_each_entry(sgl, &ctx->tsgl, list) { sg = sgl->sg; while (!sg->length) sg++; nents += sg_nents(sg); } return nents; }

Contributors

PersonTokensPropCommitsCommitProp
Tadeusz Struk4979.03%150.00%
Herbert Xu1320.97%150.00%
Total62100.00%2100.00%


static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg, int flags) { struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); struct skcipher_ctx *ctx = ask->private; struct skcipher_tfm *skc = pask->private; struct crypto_skcipher *tfm = skc->skcipher; struct skcipher_sg_list *sgl; struct scatterlist *sg; struct skcipher_async_req *sreq; struct skcipher_request *req; struct skcipher_async_rsgl *last_rsgl = NULL; unsigned int txbufs = 0, len = 0, tx_nents; unsigned int reqsize = crypto_skcipher_reqsize(tfm); unsigned int ivsize = crypto_skcipher_ivsize(tfm); int err = -ENOMEM; bool mark = false; char *iv; sreq = kzalloc(sizeof(*sreq) + reqsize + ivsize, GFP_KERNEL); if (unlikely(!sreq)) goto out; req = &sreq->req; iv = (char *)(req + 1) + reqsize; sreq->iocb = msg->msg_iocb; INIT_LIST_HEAD(&sreq->list); sreq->inflight = &ctx->inflight; lock_sock(sk); tx_nents = skcipher_all_sg_nents(ctx); sreq->tsg = kcalloc(tx_nents, sizeof(*sg), GFP_KERNEL); if (unlikely(!sreq->tsg)) goto unlock; sg_init_table(sreq->tsg, tx_nents); memcpy(iv, ctx->iv, ivsize); skcipher_request_set_tfm(req, tfm); skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, skcipher_async_cb, sreq); while (iov_iter_count(&msg->msg_iter)) { struct skcipher_async_rsgl *rsgl; int used; if (!ctx->used) { err = skcipher_wait_for_data(sk, flags); if (err) goto free; } sgl = list_first_entry(&ctx->tsgl, struct skcipher_sg_list, list); sg = sgl->sg; while (!sg->length) sg++; used = min_t(unsigned long, ctx->used, iov_iter_count(&msg->msg_iter)); used = min_t(unsigned long, used, sg->length); if (txbufs == tx_nents) { struct scatterlist *tmp; int x; /* Ran out of tx slots in async request * need to expand */ tmp = kcalloc(tx_nents * 2, sizeof(*tmp), GFP_KERNEL); if (!tmp) { err = -ENOMEM; goto free; } sg_init_table(tmp, tx_nents * 2); for (x = 0; x < tx_nents; x++) sg_set_page(&tmp[x], sg_page(&sreq->tsg[x]), sreq->tsg[x].length, sreq->tsg[x].offset); kfree(sreq->tsg); sreq->tsg = tmp; tx_nents *= 2; mark = true; } /* Need to take over the tx sgl from ctx * to the asynch req - these sgls will be freed later */ sg_set_page(sreq->tsg + txbufs++, sg_page(sg), sg->length, sg->offset); if (list_empty(&sreq->list)) { rsgl = &sreq->first_sgl; list_add_tail(&rsgl->list, &sreq->list); } else { rsgl = kmalloc(sizeof(*rsgl), GFP_KERNEL); if (!rsgl) { err = -ENOMEM; goto free