cregit-Linux how code gets into the kernel

Release 4.8 crypto/gcm.c

Directory: crypto
/*
 * GCM: Galois/Counter Mode.
 *
 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 */

#include <crypto/gf128mul.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/internal/hash.h>
#include <crypto/null.h>
#include <crypto/scatterwalk.h>
#include <crypto/hash.h>
#include "internal.h"
#include <linux/completion.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>


struct gcm_instance_ctx {
	
struct crypto_skcipher_spawn ctr;
	
struct crypto_ahash_spawn ghash;
};


struct crypto_gcm_ctx {
	
struct crypto_skcipher *ctr;
	
struct crypto_ahash *ghash;
};


struct crypto_rfc4106_ctx {
	
struct crypto_aead *child;
	
u8 nonce[4];
};


struct crypto_rfc4106_req_ctx {
	
struct scatterlist src[3];
	
struct scatterlist dst[3];
	
struct aead_request subreq;
};


struct crypto_rfc4543_instance_ctx {
	
struct crypto_aead_spawn aead;
};


struct crypto_rfc4543_ctx {
	
struct crypto_aead *child;
	
struct crypto_skcipher *null;
	
u8 nonce[4];
};


struct crypto_rfc4543_req_ctx {
	
struct aead_request subreq;
};


struct crypto_gcm_ghash_ctx {
	
unsigned int cryptlen;
	
struct scatterlist *src;
	
int (*complete)(struct aead_request *req, u32 flags);
};


struct crypto_gcm_req_priv_ctx {
	
u8 iv[16];
	
u8 auth_tag[16];
	
u8 iauth_tag[16];
	
struct scatterlist src[3];
	
struct scatterlist dst[3];
	
struct scatterlist sg;
	
struct crypto_gcm_ghash_ctx ghash_ctx;
	union {
		
struct ahash_request ahreq;
		
struct skcipher_request skreq;
	} 
u;
};


struct crypto_gcm_setkey_result {
	
int err;
	
struct completion completion;
};

static struct {
	
u8 buf[16];
	
struct scatterlist sg;
} 
*gcm_zeroes;

static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);


static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( struct aead_request *req) { unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu47100.00%1100.00%
Total47100.00%1100.00%


static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err) { struct crypto_gcm_setkey_result *result = req->data; if (err == -EINPROGRESS) return; result->err = err; complete(&result->completion); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying2862.22%150.00%
mikko herranenmikko herranen1737.78%150.00%
Total45100.00%2100.00%


static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_ahash *ghash = ctx->ghash; struct crypto_skcipher *ctr = ctx->ctr; struct { be128 hash; u8 iv[8]; struct crypto_gcm_setkey_result result; struct scatterlist sg[1]; struct skcipher_request req; } *data; int err; crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & CRYPTO_TFM_REQ_MASK); err = crypto_skcipher_setkey(ctr, key, keylen); crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) & CRYPTO_TFM_RES_MASK); if (err) return err; data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr), GFP_KERNEL); if (!data) return -ENOMEM; init_completion(&data->result.completion); sg_init_one(data->sg, &data->hash, sizeof(data->hash)); skcipher_request_set_tfm(&data->req, ctr); skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_gcm_setkey_done, &data->result); skcipher_request_set_crypt(&data->req, data->sg, data->sg, sizeof(data->hash), data->iv); err = crypto_skcipher_encrypt(&data->req); if (err == -EINPROGRESS || err == -EBUSY) { err = wait_for_completion_interruptible( &data->result.completion); if (!err) err = data->result.err; } if (err) goto out; crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & CRYPTO_TFM_REQ_MASK); err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) & CRYPTO_TFM_RES_MASK); out: kzfree(data); return err; }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying30384.17%125.00%
mikko herranenmikko herranen3810.56%125.00%
herbert xuherbert xu195.28%250.00%
Total360100.00%4100.00%


static int crypto_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { switch (authsize) { case 4: case 8: case 12: case 13: case 14: case 15: case 16: break; default: return -EINVAL; } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying51100.00%1100.00%
Total51100.00%1100.00%


static void crypto_gcm_init_common(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); __be32 counter = cpu_to_be32(1); struct scatterlist *sg; memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); memcpy(pctx->iv, req->iv, 12); memcpy(pctx->iv + 12, &counter, 4); sg_init_table(pctx->src, 3); sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen); if (sg != pctx->src + 1) sg_chain(pctx->src, 2, sg); if (req->src != req->dst) { sg_init_table(pctx->dst, 3); sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen); if (sg != pctx->dst + 1) sg_chain(pctx->dst, 2, sg); } }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying15669.96%125.00%
herbert xuherbert xu6428.70%125.00%
dan williamsdan williams20.90%125.00%
mikko herranenmikko herranen10.45%125.00%
Total223100.00%4100.00%


static void crypto_gcm_init_crypt(struct aead_request *req, unsigned int cryptlen) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct skcipher_request *skreq = &pctx->u.skreq; struct scatterlist *dst; dst = req->src == req->dst ? pctx->src : pctx->dst; skcipher_request_set_tfm(skreq, ctx->ctr); skcipher_request_set_crypt(skreq, pctx->src, dst, cryptlen + sizeof(pctx->auth_tag), pctx->iv); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu10592.92%250.00%
huang yinghuang ying76.19%125.00%
mikko herranenmikko herranen10.88%125.00%
Total113100.00%4100.00%


static inline unsigned int gcm_remain(unsigned int len) { len &= 0xfU; return len ? 16 - len : 0; }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu2184.00%150.00%
huang yinghuang ying416.00%150.00%
Total25100.00%2100.00%

static void gcm_hash_len_done(struct crypto_async_request *areq, int err);
static int gcm_hash_update(struct aead_request *req, crypto_completion_t compl, struct scatterlist *src, unsigned int len, u32 flags) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct ahash_request *ahreq = &pctx->u.ahreq; ahash_request_set_callback(ahreq, flags, compl, req); ahash_request_set_crypt(ahreq, src, NULL, len); return crypto_ahash_update(ahreq); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying3444.74%125.00%
mikko herranenmikko herranen2634.21%125.00%
herbert xuherbert xu1418.42%125.00%
mark d. rustadmark d. rustad22.63%125.00%
Total76100.00%4100.00%


static int gcm_hash_remain(struct aead_request *req, unsigned int remain, crypto_completion_t compl, u32 flags) { return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying2565.79%125.00%
herbert xuherbert xu923.68%125.00%
mark d. rustadmark d. rustad25.26%125.00%
mikko herranenmikko herranen25.26%125.00%
Total38100.00%4100.00%


static int gcm_hash_len(struct aead_request *req, u32 flags) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct ahash_request *ahreq = &pctx->u.ahreq; struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; u128 lengths; lengths.a = cpu_to_be64(req->assoclen * 8); lengths.b = cpu_to_be64(gctx->cryptlen * 8); memcpy(pctx->iauth_tag, &lengths, 16); sg_init_one(&pctx->sg, pctx->iauth_tag, 16); ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req); ahash_request_set_crypt(ahreq, &pctx->sg, pctx->iauth_tag, sizeof(lengths)); return crypto_ahash_finup(ahreq); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying10979.56%133.33%
herbert xuherbert xu1913.87%133.33%
mikko herranenmikko herranen96.57%133.33%
Total137100.00%3100.00%


static int gcm_hash_len_continue(struct aead_request *req, u32 flags) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; return gctx->complete(req, flags); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying2556.82%250.00%
herbert xuherbert xu1534.09%125.00%
mikko herranenmikko herranen49.09%125.00%
Total44100.00%4100.00%


static void gcm_hash_len_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; if (err) goto out; err = gcm_hash_len_continue(req, 0); if (err == -EINPROGRESS) return; out: aead_request_complete(req, err); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying2442.86%250.00%
mikko herranenmikko herranen2341.07%125.00%
herbert xuherbert xu916.07%125.00%
Total56100.00%4100.00%


static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags) { return gcm_hash_len(req, flags) ?: gcm_hash_len_continue(req, flags); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu1756.67%133.33%
mikko herranenmikko herranen826.67%133.33%
huang yinghuang ying516.67%133.33%
Total30100.00%3100.00%


static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; if (err) goto out; err = gcm_hash_crypt_remain_continue(req, 0); if (err == -EINPROGRESS) return; out: aead_request_complete(req, err); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying2850.00%240.00%
herbert xuherbert xu2035.71%240.00%
mikko herranenmikko herranen814.29%120.00%
Total56100.00%5100.00%


static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; unsigned int remain; remain = gcm_remain(gctx->cryptlen); if (remain) return gcm_hash_remain(req, remain, gcm_hash_crypt_remain_done, flags) ?: gcm_hash_crypt_remain_continue(req, flags); return gcm_hash_crypt_remain_continue(req, flags); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying3645.57%250.00%
herbert xuherbert xu2227.85%125.00%
mikko herranenmikko herranen2126.58%125.00%
Total79100.00%4100.00%


static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; if (err) goto out; err = gcm_hash_crypt_continue(req, 0); if (err == -EINPROGRESS) return; out: aead_request_complete(req, err); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu2850.00%250.00%
huang yinghuang ying2850.00%250.00%
Total56100.00%4100.00%


static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; if (gctx->cryptlen) return gcm_hash_update(req, gcm_hash_crypt_done, gctx->src, gctx->cryptlen, flags) ?: gcm_hash_crypt_continue(req, flags); return gcm_hash_crypt_remain_continue(req, flags); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying4358.11%250.00%
herbert xuherbert xu3141.89%250.00%
Total74100.00%4100.00%


static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; if (err) goto out; err = gcm_hash_assoc_remain_continue(req, 0); if (err == -EINPROGRESS) return; out: aead_request_complete(req, err); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying2850.00%250.00%
herbert xuherbert xu2748.21%125.00%
mikko herranenmikko herranen11.79%125.00%
Total56100.00%4100.00%


static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags) { unsigned int remain; remain = gcm_remain(req->assoclen); if (remain) return gcm_hash_remain(req, remain, gcm_hash_assoc_remain_done, flags) ?: gcm_hash_assoc_remain_continue(req, flags); return gcm_hash_assoc_remain_continue(req, flags); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying2745.76%240.00%
herbert xuherbert xu2542.37%240.00%
mikko herranenmikko herranen711.86%120.00%
Total59100.00%5100.00%


static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; if (err) goto out; err = gcm_hash_assoc_continue(req, 0); if (err == -EINPROGRESS) return; out: aead_request_complete(req, err); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu3562.50%250.00%
huang yinghuang ying2137.50%250.00%
Total56100.00%4100.00%


static int gcm_hash_init_continue(struct aead_request *req, u32 flags) { if (req->assoclen) return gcm_hash_update(req, gcm_hash_assoc_done, req->src, req->assoclen, flags) ?: gcm_hash_assoc_continue(req, flags); return gcm_hash_assoc_remain_continue(req, flags); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying3462.96%250.00%
herbert xuherbert xu2037.04%250.00%
Total54100.00%4100.00%


static void gcm_hash_init_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; if (err) goto out; err = gcm_hash_init_continue(req, 0); if (err == -EINPROGRESS) return; out: aead_request_complete(req, err); }

Contributors

PersonTokensPropCommitsCommitProp
huang yinghuang ying2850.00%250.00%
herbert xuherbert xu2850.00%250.00%
Total56100.00%4100.00%


static int gcm_hash(struct aead_request *req, u32 flags) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct ahash_request *ahreq = &pctx->u.ahreq; struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); ahash_request_set_tfm(ahreq, ctx->ghash); ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req); return crypto_ahash_init(ahreq) ?: gcm_hash_init_continue(req, flags); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu3440.96%250.00%
huang yinghuang ying2530.12%125.00%
mikko herranenmikko herranen2428.92%125.00%
Total83100.00%4100.00%


static int gcm_enc_copy_hash(struct aead_request *req, u32 flags) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); u8 *auth_tag = pctx->auth_tag; crypto_xor(auth_tag, pctx->iauth_tag, 16); scatterwalk_map_and_copy(auth_tag, req->dst, req->assoclen + req->cryptlen, crypto_aead_authsize(aead), 1); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu6480.00%250.00%
huang yinghuang ying911.25%125.00%
mikko herranenmikko herranen78.75%125.00%
Total80100.00%4100.00%


static int gcm_encrypt_continue(struct aead_request *req, u32 flags) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst); gctx->cryptlen = req->cryptlen; gctx->complete = gcm_enc_copy_hash; return gcm_hash(req, flags); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu6379.75%250.00%
huang yinghuang ying1113.92%125.00%
mikko herranenmikko herranen56.33%125.00%
Total79100.00%4100.00%


static void gcm_encrypt_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; if (err) goto out; err = gcm_encrypt_continue(req, 0); if (err == -EINPROGRESS) return; out: aead_request_complete(req, err); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu3766.07%133.33%
huang yinghuang ying1425.00%133.33%
mikko herranenmikko herranen58.93%133.33%
Total56100.00%3100.00%


static int crypto_gcm_encrypt(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct skcipher_request *skreq = &pctx->u.skreq; u32 flags = aead_request_flags(req); crypto_gcm_init_common(req); crypto_gcm_init_crypt(req, req->cryptlen); skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req); return crypto_skcipher_encrypt(skreq) ?: gcm_encrypt_continue(req, flags); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu5366.25%360.00%
mikko herranenmikko herranen2328.75%120.00%
huang yinghuang ying45.00%120.00%
Total80100.00%5100.00%


static int crypto_gcm_verify(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); u8 *auth_tag = pctx->auth_tag; u8 *iauth_tag = pctx->iauth_tag; unsigned int authsize = crypto_aead_authsize(aead); unsigned int cryptlen = req->cryptlen - authsize; crypto_xor(auth_tag, iauth_tag, 16); scatterwalk_map_and_copy(iauth_tag, req->src, req->assoclen + cryptlen, authsize, 0); return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu8779.82%240.00%
huang yinghuang ying2018.35%240.00%
mikko herranenmikko herranen21.83%120.00%
Total109100.00%5100.00%


static void gcm_decrypt_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; if (!err) err = crypto_gcm_verify(req); aead_request_complete(req, err); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu3685.71%375.00%
huang yinghuang ying614.29%125.00%
Total42100.00%4100.00%


static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct skcipher_request *skreq = &pctx->u.skreq; struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; crypto_gcm_init_crypt(req, gctx->cryptlen); skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req); return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu6279.49%466.67%
huang yinghuang ying1417.95%116.67%
mikko herranenmikko herranen22.56%116.67%
Total78100.00%6100.00%


static int crypto_gcm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; unsigned int authsize = crypto_aead_authsize(aead); unsigned int cryptlen = req->cryptlen; u32 flags = aead_request_flags(req); cryptlen -= authsize; crypto_gcm_init_common(req); gctx->src = sg_next(pctx->src); gctx->cryptlen = cryptlen; gctx->complete = gcm_dec_hash_continue; return gcm_hash(req, flags); }

Contributors

PersonTokensPropCommitsCommitProp
herbert xuherbert xu8681.13%375.00%
mikko herranenmikko herranen2018.87%125.00%
Total106100.00%4100.00%


static int crypto_gcm_init_tfm(struct crypto_aead *tfm) { struct aead_instance *inst = aead_alg_instance(tfm); struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_skcipher *ctr; struct crypto_ahash *ghash; unsigned long align; int err; ghash = crypto_spawn_ahash(&ictx->ghash); if (IS_ERR(ghash)) return PTR_ERR(ghash); ctr = crypto_spawn_skcipher2(&ictx->ctr); err = PTR_ERR(ctr); if (IS_ERR(ctr)) goto err_free_hash; ctx->ctr = ctr; ctx->ghash = ghash; align = crypto_aead_alignmask(tfm); align &= ~(crypto_tfm_ctx_alignment() - 1); crypto_aead_set_reqsize(