| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Ryder Lee | 6571 | 99.85% | 9 | 69.23% |
| Corentin Labbe | 5 | 0.08% | 1 | 7.69% |
| Thomas Gleixner | 2 | 0.03% | 1 | 7.69% |
| Arnd Bergmann | 2 | 0.03% | 1 | 7.69% |
| Colin Ian King | 1 | 0.02% | 1 | 7.69% |
| Total | 6581 | 13 |
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353
// SPDX-License-Identifier: GPL-2.0-only /* * Cryptographic API. * * Driver for EIP97 SHA1/SHA2(HMAC) acceleration. * * Copyright (c) 2016 Ryder Lee <ryder.lee@mediatek.com> * * Some ideas are from atmel-sha.c and omap-sham.c drivers. */ #include <crypto/hmac.h> #include <crypto/sha.h> #include "mtk-platform.h" #define SHA_ALIGN_MSK (sizeof(u32) - 1) #define SHA_QUEUE_SIZE 512 #define SHA_BUF_SIZE ((u32)PAGE_SIZE) #define SHA_OP_UPDATE 1 #define SHA_OP_FINAL 2 #define SHA_DATA_LEN_MSK cpu_to_le32(GENMASK(16, 0)) #define SHA_MAX_DIGEST_BUF_SIZE 32 /* SHA command token */ #define SHA_CT_SIZE 5 #define SHA_CT_CTRL_HDR cpu_to_le32(0x02220000) #define SHA_CMD0 cpu_to_le32(0x03020000) #define SHA_CMD1 cpu_to_le32(0x21060000) #define SHA_CMD2 cpu_to_le32(0xe0e63802) /* SHA transform information */ #define SHA_TFM_HASH cpu_to_le32(0x2 << 0) #define SHA_TFM_SIZE(x) cpu_to_le32((x) << 8) #define SHA_TFM_START cpu_to_le32(0x1 << 4) #define SHA_TFM_CONTINUE cpu_to_le32(0x1 << 5) #define SHA_TFM_HASH_STORE cpu_to_le32(0x1 << 19) #define SHA_TFM_SHA1 cpu_to_le32(0x2 << 23) #define SHA_TFM_SHA256 cpu_to_le32(0x3 << 23) #define SHA_TFM_SHA224 cpu_to_le32(0x4 << 23) #define SHA_TFM_SHA512 cpu_to_le32(0x5 << 23) #define SHA_TFM_SHA384 cpu_to_le32(0x6 << 23) #define SHA_TFM_DIGEST(x) cpu_to_le32(((x) & GENMASK(3, 0)) << 24) /* SHA flags */ #define SHA_FLAGS_BUSY BIT(0) #define SHA_FLAGS_FINAL BIT(1) #define SHA_FLAGS_FINUP BIT(2) #define SHA_FLAGS_SG BIT(3) #define SHA_FLAGS_ALGO_MSK GENMASK(8, 4) #define SHA_FLAGS_SHA1 BIT(4) #define SHA_FLAGS_SHA224 BIT(5) #define SHA_FLAGS_SHA256 BIT(6) #define SHA_FLAGS_SHA384 BIT(7) #define SHA_FLAGS_SHA512 BIT(8) #define SHA_FLAGS_HMAC BIT(9) #define SHA_FLAGS_PAD BIT(10) /** * mtk_sha_info - hardware information of AES * @cmd: command token, hardware instruction * @tfm: transform state of cipher algorithm. * @state: contains keys and initial vectors. * */ struct mtk_sha_info { __le32 ctrl[2]; __le32 cmd[3]; __le32 tfm[2]; __le32 digest[SHA_MAX_DIGEST_BUF_SIZE]; }; struct mtk_sha_reqctx { struct mtk_sha_info info; unsigned long flags; unsigned long op; u64 digcnt; size_t bufcnt; dma_addr_t dma_addr; __le32 ct_hdr; u32 ct_size; dma_addr_t ct_dma; dma_addr_t tfm_dma; /* Walk state */ struct scatterlist *sg; u32 offset; /* Offset in current sg */ u32 total; /* Total request */ size_t ds; size_t bs; u8 *buffer; }; struct mtk_sha_hmac_ctx { struct crypto_shash *shash; u8 ipad[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); u8 opad[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); }; struct mtk_sha_ctx { struct mtk_cryp *cryp; unsigned long flags; u8 id; u8 buf[SHA_BUF_SIZE] __aligned(sizeof(u32)); struct mtk_sha_hmac_ctx base[0]; }; struct mtk_sha_drv { struct list_head dev_list; /* Device list lock */ spinlock_t lock; }; static struct mtk_sha_drv mtk_sha = { .dev_list = LIST_HEAD_INIT(mtk_sha.dev_list), .lock = __SPIN_LOCK_UNLOCKED(mtk_sha.lock), }; static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id, struct ahash_request *req); static inline u32 mtk_sha_read(struct mtk_cryp *cryp, u32 offset) { return readl_relaxed(cryp->base + offset); } static inline void mtk_sha_write(struct mtk_cryp *cryp, u32 offset, u32 value) { writel_relaxed(value, cryp->base + offset); } static inline void mtk_sha_ring_shift(struct mtk_ring *ring, struct mtk_desc **cmd_curr, struct mtk_desc **res_curr, int *count) { *cmd_curr = ring->cmd_next++; *res_curr = ring->res_next++; (*count)++; if (ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) { ring->cmd_next = ring->cmd_base; ring->res_next = ring->res_base; } } static struct mtk_cryp *mtk_sha_find_dev(struct mtk_sha_ctx *tctx) { struct mtk_cryp *cryp = NULL; struct mtk_cryp *tmp; spin_lock_bh(&mtk_sha.lock); if (!tctx->cryp) { list_for_each_entry(tmp, &mtk_sha.dev_list, sha_list) { cryp = tmp; break; } tctx->cryp = cryp; } else { cryp = tctx->cryp; } /* * Assign record id to tfm in round-robin fashion, and this * will help tfm to bind to corresponding descriptor rings. */ tctx->id = cryp->rec; cryp->rec = !cryp->rec; spin_unlock_bh(&mtk_sha.lock); return cryp; } static int mtk_sha_append_sg(struct mtk_sha_reqctx *ctx) { size_t count; while ((ctx->bufcnt < SHA_BUF_SIZE) && ctx->total) { count = min(ctx->sg->length - ctx->offset, ctx->total); count = min(count, SHA_BUF_SIZE - ctx->bufcnt); if (count <= 0) { /* * Check if count <= 0 because the buffer is full or * because the sg length is 0. In the latest case, * check if there is another sg in the list, a 0 length * sg doesn't necessarily mean the end of the sg list. */ if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) { ctx->sg = sg_next(ctx->sg); continue; } else { break; } } scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg, ctx->offset, count, 0); ctx->bufcnt += count; ctx->offset += count; ctx->total -= count; if (ctx->offset == ctx->sg->length) { ctx->sg = sg_next(ctx->sg); if (ctx->sg) ctx->offset = 0; else ctx->total = 0; } } return 0; } /* * The purpose of this padding is to ensure that the padded message is a * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). * The bit "1" is appended at the end of the message followed by * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or * 128 bits block (SHA384/SHA512) equals to the message length in bits * is appended. * * For SHA1/SHA224/SHA256, padlen is calculated as followed: * - if message length < 56 bytes then padlen = 56 - message length * - else padlen = 64 + 56 - message length * * For SHA384/SHA512, padlen is calculated as followed: * - if message length < 112 bytes then padlen = 112 - message length * - else padlen = 128 + 112 - message length */ static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) { u32 index, padlen; u64 bits[2]; u64 size = ctx->digcnt; size += ctx->bufcnt; size += len; bits[1] = cpu_to_be64(size << 3); bits[0] = cpu_to_be64(size >> 61); switch (ctx->flags & SHA_FLAGS_ALGO_MSK) { case SHA_FLAGS_SHA384: case SHA_FLAGS_SHA512: index = ctx->bufcnt & 0x7f; padlen = (index < 112) ? (112 - index) : ((128 + 112) - index); *(ctx->buffer + ctx->bufcnt) = 0x80; memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen - 1); memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); ctx->bufcnt += padlen + 16; ctx->flags |= SHA_FLAGS_PAD; break; default: index = ctx->bufcnt & 0x3f; padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); *(ctx->buffer + ctx->bufcnt) = 0x80; memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen - 1); memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); ctx->bufcnt += padlen + 8; ctx->flags |= SHA_FLAGS_PAD; break; } } /* Initialize basic transform information of SHA */ static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx) { struct mtk_sha_info *info = &ctx->info; ctx->ct_hdr = SHA_CT_CTRL_HDR; ctx->ct_size = SHA_CT_SIZE; info->tfm[0] = SHA_TFM_HASH | SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds)); switch (ctx->flags & SHA_FLAGS_ALGO_MSK) { case SHA_FLAGS_SHA1: info->tfm[0] |= SHA_TFM_SHA1; break; case SHA_FLAGS_SHA224: info->tfm[0] |= SHA_TFM_SHA224; break; case SHA_FLAGS_SHA256: info->tfm[0] |= SHA_TFM_SHA256; break; case SHA_FLAGS_SHA384: info->tfm[0] |= SHA_TFM_SHA384; break; case SHA_FLAGS_SHA512: info->tfm[0] |= SHA_TFM_SHA512; break; default: /* Should not happen... */ return; } info->tfm[1] = SHA_TFM_HASH_STORE; info->ctrl[0] = info->tfm[0] | SHA_TFM_CONTINUE | SHA_TFM_START; info->ctrl[1] = info->tfm[1]; info->cmd[0] = SHA_CMD0; info->cmd[1] = SHA_CMD1; info->cmd[2] = SHA_CMD2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds)); } /* * Update input data length field of transform information and * map it to DMA region. */ static int mtk_sha_info_update(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, size_t len1, size_t len2) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); struct mtk_sha_info *info = &ctx->info; ctx->ct_hdr &= ~SHA_DATA_LEN_MSK; ctx->ct_hdr |= cpu_to_le32(len1 + len2); info->cmd[0] &= ~SHA_DATA_LEN_MSK; info->cmd[0] |= cpu_to_le32(len1 + len2); /* Setting SHA_TFM_START only for the first iteration */ if (ctx->digcnt) info->ctrl[0] &= ~SHA_TFM_START; ctx->digcnt += len1; ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info), DMA_BIDIRECTIONAL); if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) { dev_err(cryp->dev, "dma %zu bytes error\n", sizeof(*info)); return -EINVAL; } ctx->tfm_dma = ctx->ct_dma + sizeof(info->ctrl) + sizeof(info->cmd); return 0; } /* * Because of hardware limitation, we must pre-calculate the inner * and outer digest that need to be processed firstly by engine, then * apply the result digest to the input message. These complex hashing * procedures limits HMAC performance, so we use fallback SW encoding. */ static int mtk_sha_finish_hmac(struct ahash_request *req) { struct mtk_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); struct mtk_sha_hmac_ctx *bctx = tctx->base; struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); SHASH_DESC_ON_STACK(shash, bctx->shash); shash->tfm = bctx->shash; return crypto_shash_init(shash) ?: crypto_shash_update(shash, bctx->opad, ctx->bs) ?: crypto_shash_finup(shash, req->result, ctx->ds, req->result); } /* Initialize request context */ static int mtk_sha_init(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm); struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); ctx->flags = 0; ctx->ds = crypto_ahash_digestsize(tfm); switch (ctx->ds) { case SHA1_DIGEST_SIZE: ctx->flags |= SHA_FLAGS_SHA1; ctx->bs = SHA1_BLOCK_SIZE; break; case SHA224_DIGEST_SIZE: ctx->flags |= SHA_FLAGS_SHA224; ctx->bs = SHA224_BLOCK_SIZE; break; case SHA256_DIGEST_SIZE: ctx->flags |= SHA_FLAGS_SHA256; ctx->bs = SHA256_BLOCK_SIZE; break; case SHA384_DIGEST_SIZE: ctx->flags |= SHA_FLAGS_SHA384; ctx->bs = SHA384_BLOCK_SIZE; break; case SHA512_DIGEST_SIZE: ctx->flags |= SHA_FLAGS_SHA512; ctx->bs = SHA512_BLOCK_SIZE; break; default: return -EINVAL; } ctx->bufcnt = 0; ctx->digcnt = 0; ctx->buffer = tctx->buf; if (tctx->flags & SHA_FLAGS_HMAC) { struct mtk_sha_hmac_ctx *bctx = tctx->base; memcpy(ctx->buffer, bctx->ipad, ctx->bs); ctx->bufcnt = ctx->bs; ctx->flags |= SHA_FLAGS_HMAC; } return 0; } static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, dma_addr_t addr1, size_t len1, dma_addr_t addr2, size_t len2) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); struct mtk_ring *ring = cryp->ring[sha->id]; struct mtk_desc *cmd, *res; int err, count = 0; err = mtk_sha_info_update(cryp, sha, len1, len2); if (err) return err; /* Fill in the command/result descriptors */ mtk_sha_ring_shift(ring, &cmd, &res, &count); res->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1); cmd->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1) | MTK_DESC_CT_LEN(ctx->ct_size); cmd->buf = cpu_to_le32(addr1); cmd->ct = cpu_to_le32(ctx->ct_dma); cmd->ct_hdr = ctx->ct_hdr; cmd->tfm = cpu_to_le32(ctx->tfm_dma); if (len2) { mtk_sha_ring_shift(ring, &cmd, &res, &count); res->hdr = MTK_DESC_BUF_LEN(len2); cmd->hdr = MTK_DESC_BUF_LEN(len2); cmd->buf = cpu_to_le32(addr2); } cmd->hdr |= MTK_DESC_LAST; res->hdr |= MTK_DESC_LAST; /* * Make sure that all changes to the DMA ring are done before we * start engine. */ wmb(); /* Start DMA transfer */ mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count)); mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count)); return -EINPROGRESS; } static int mtk_sha_dma_map(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, struct mtk_sha_reqctx *ctx, size_t count) { ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer, SHA_BUF_SIZE, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) { dev_err(cryp->dev, "dma map error\n"); return -EINVAL; } ctx->flags &= ~SHA_FLAGS_SG; return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count, 0, 0); } static int mtk_sha_update_slow(struct mtk_cryp *cryp, struct mtk_sha_rec *sha) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); size_t count; u32 final; mtk_sha_append_sg(ctx); final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; dev_dbg(cryp->dev, "slow: bufcnt: %zu\n", ctx->bufcnt); if (final) { sha->flags |= SHA_FLAGS_FINAL; mtk_sha_fill_padding(ctx, 0); } if (final || (ctx->bufcnt == SHA_BUF_SIZE && ctx->total)) { count = ctx->bufcnt; ctx->bufcnt = 0; return mtk_sha_dma_map(cryp, sha, ctx, count); } return 0; } static int mtk_sha_update_start(struct mtk_cryp *cryp, struct mtk_sha_rec *sha) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); u32 len, final, tail; struct scatterlist *sg; if (!ctx->total) return 0; if (ctx->bufcnt || ctx->offset) return mtk_sha_update_slow(cryp, sha); sg = ctx->sg; if (!IS_ALIGNED(sg->offset, sizeof(u32))) return mtk_sha_update_slow(cryp, sha); if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->bs)) /* size is not ctx->bs aligned */ return mtk_sha_update_slow(cryp, sha); len = min(ctx->total, sg->length); if (sg_is_last(sg)) { if (!(ctx->flags & SHA_FLAGS_FINUP)) { /* not last sg must be ctx->bs aligned */ tail = len & (ctx->bs - 1); len -= tail; } } ctx->total -= len; ctx->offset = len; /* offset where to start slow */ final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; /* Add padding */ if (final) { size_t count; tail = len & (ctx->bs - 1); len -= tail; ctx->total += tail; ctx->offset = len; /* offset where to start slow */ sg = ctx->sg; mtk_sha_append_sg(ctx); mtk_sha_fill_padding(ctx, len); ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer, SHA_BUF_SIZE, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) { dev_err(cryp->dev, "dma map bytes error\n"); return -EINVAL; } sha->flags |= SHA_FLAGS_FINAL; count = ctx->bufcnt; ctx->bufcnt = 0; if (len == 0) { ctx->flags &= ~SHA_FLAGS_SG; return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count, 0, 0); } else { ctx->sg = sg; if (!dma_map_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE)) { dev_err(cryp->dev, "dma_map_sg error\n"); return -EINVAL; } ctx->flags |= SHA_FLAGS_SG; return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), len, ctx->dma_addr, count); } } if (!dma_map_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE)) { dev_err(cryp->dev, "dma_map_sg error\n"); return -EINVAL; } ctx->flags |= SHA_FLAGS_SG; return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), len, 0, 0); } static int mtk_sha_final_req(struct mtk_cryp *cryp, struct mtk_sha_rec *sha) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); size_t count; mtk_sha_fill_padding(ctx, 0); sha->flags |= SHA_FLAGS_FINAL; count = ctx->bufcnt; ctx->bufcnt = 0; return mtk_sha_dma_map(cryp, sha, ctx, count); } /* Copy ready hash (+ finalize hmac) */ static int mtk_sha_finish(struct ahash_request *req) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); __le32 *digest = ctx->info.digest; u32 *result = (u32 *)req->result; int i; /* Get the hash from the digest buffer */ for (i = 0; i < SIZE_IN_WORDS(ctx->ds); i++) result[i] = le32_to_cpu(digest[i]); if (ctx->flags & SHA_FLAGS_HMAC) return mtk_sha_finish_hmac(req); return 0; } static void mtk_sha_finish_req(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, int err) { if (likely(!err && (SHA_FLAGS_FINAL & sha->flags))) err = mtk_sha_finish(sha->req); sha->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL); sha->req->base.complete(&sha->req->base, err); /* Handle new request */ tasklet_schedule(&sha->queue_task); } static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id, struct ahash_request *req) { struct mtk_sha_rec *sha = cryp->sha[id]; struct crypto_async_request *async_req, *backlog; struct mtk_sha_reqctx *ctx; unsigned long flags; int err = 0, ret = 0; spin_lock_irqsave(&sha->lock, flags); if (req) ret = ahash_enqueue_request(&sha->queue, req); if (SHA_FLAGS_BUSY & sha->flags) { spin_unlock_irqrestore(&sha->lock, flags); return ret; } backlog = crypto_get_backlog(&sha->queue); async_req = crypto_dequeue_request(&sha->queue); if (async_req) sha->flags |= SHA_FLAGS_BUSY; spin_unlock_irqrestore(&sha->lock, flags); if (!async_req) return ret; if (backlog) backlog->complete(backlog, -EINPROGRESS); req = ahash_request_cast(async_req); ctx = ahash_request_ctx(req); sha->req = req; mtk_sha_info_init(ctx); if (ctx->op == SHA_OP_UPDATE) { err = mtk_sha_update_start(cryp, sha); if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP)) /* No final() after finup() */ err = mtk_sha_final_req(cryp, sha); } else if (ctx->op == SHA_OP_FINAL) { err = mtk_sha_final_req(cryp, sha); } if (unlikely(err != -EINPROGRESS)) /* Task will not finish it, so do it here */ mtk_sha_finish_req(cryp, sha, err); return ret; } static int mtk_sha_enqueue(struct ahash_request *req, u32 op) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); struct mtk_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); ctx->op = op; return mtk_sha_handle_queue(tctx->cryp, tctx->id, req); } static void mtk_sha_unmap(struct mtk_cryp *cryp, struct mtk_sha_rec *sha) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info), DMA_BIDIRECTIONAL); if (ctx->flags & SHA_FLAGS_SG) { dma_unmap_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE); if (ctx->sg->length == ctx->offset) { ctx->sg = sg_next(ctx->sg); if (ctx->sg) ctx->offset = 0; } if (ctx->flags & SHA_FLAGS_PAD) { dma_unmap_single(cryp->dev, ctx->dma_addr, SHA_BUF_SIZE, DMA_TO_DEVICE); } } else dma_unmap_single(cryp->dev, ctx->dma_addr, SHA_BUF_SIZE, DMA_TO_DEVICE); } static void mtk_sha_complete(struct mtk_cryp *cryp, struct mtk_sha_rec *sha) { int err = 0; err = mtk_sha_update_start(cryp, sha); if (err != -EINPROGRESS) mtk_sha_finish_req(cryp, sha, err); } static int mtk_sha_update(struct ahash_request *req) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); ctx->total = req->nbytes; ctx->sg = req->src; ctx->offset = 0; if ((ctx->bufcnt + ctx->total < SHA_BUF_SIZE) && !(ctx->flags & SHA_FLAGS_FINUP)) return mtk_sha_append_sg(ctx); return mtk_sha_enqueue(req, SHA_OP_UPDATE); } static int mtk_sha_final(struct ahash_request *req) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); ctx->flags |= SHA_FLAGS_FINUP; if (ctx->flags & SHA_FLAGS_PAD) return mtk_sha_finish(req); return mtk_sha_enqueue(req, SHA_OP_FINAL); } static int mtk_sha_finup(struct ahash_request *req) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); int err1, err2; ctx->flags |= SHA_FLAGS_FINUP; err1 = mtk_sha_update(req); if (err1 == -EINPROGRESS || err1 == -EBUSY) return err1; /* * final() has to be always called to cleanup resources * even if update() failed */ err2 = mtk_sha_final(req); return err1 ?: err2; } static int mtk_sha_digest(struct ahash_request *req) { return mtk_sha_init(req) ?: mtk_sha_finup(req); } static int mtk_sha_setkey(struct crypto_ahash *tfm, const u8 *key, u32 keylen) { struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm); struct mtk_sha_hmac_ctx *bctx = tctx->base; size_t bs = crypto_shash_blocksize(bctx->shash); size_t ds = crypto_shash_digestsize(bctx->shash); int err, i; SHASH_DESC_ON_STACK(shash, bctx->shash); shash->tfm = bctx->shash; if (keylen > bs) { err = crypto_shash_digest(shash, key, keylen, bctx->ipad); if (err) return err; keylen = ds; } else { memcpy(bctx->ipad, key, keylen); } memset(bctx->ipad + keylen, 0, bs - keylen); memcpy(bctx->opad, bctx->ipad, bs); for (i = 0; i < bs; i++) { bctx->ipad[i] ^= HMAC_IPAD_VALUE; bctx->opad[i] ^= HMAC_OPAD_VALUE; } return 0; } static int mtk_sha_export(struct ahash_request *req, void *out) { const struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); memcpy(out, ctx, sizeof(*ctx)); return 0; } static int mtk_sha_import(struct ahash_request *req, const void *in) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); memcpy(ctx, in, sizeof(*ctx)); return 0; } static int mtk_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) { struct mtk_sha_ctx *tctx = crypto_tfm_ctx(tfm); struct mtk_cryp *cryp = NULL; cryp = mtk_sha_find_dev(tctx); if (!cryp) return -ENODEV; crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct mtk_sha_reqctx)); if (alg_base) { struct mtk_sha_hmac_ctx *bctx = tctx->base; tctx->flags |= SHA_FLAGS_HMAC; bctx->shash = crypto_alloc_shash(alg_base, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(bctx->shash)) { pr_err("base driver %s could not be loaded.\n", alg_base); return PTR_ERR(bctx->shash); } } return 0; } static int mtk_sha_cra_init(struct crypto_tfm *tfm) { return mtk_sha_cra_init_alg(tfm, NULL); } static int mtk_sha_cra_sha1_init(struct crypto_tfm *tfm) { return mtk_sha_cra_init_alg(tfm, "sha1"); } static int mtk_sha_cra_sha224_init(struct crypto_tfm *tfm) { return mtk_sha_cra_init_alg(tfm, "sha224"); } static int mtk_sha_cra_sha256_init(struct crypto_tfm *tfm) { return mtk_sha_cra_init_alg(tfm, "sha256"); } static int mtk_sha_cra_sha384_init(struct crypto_tfm *tfm) { return mtk_sha_cra_init_alg(tfm, "sha384"); } static int mtk_sha_cra_sha512_init(struct crypto_tfm *tfm) { return mtk_sha_cra_init_alg(tfm, "sha512"); } static void mtk_sha_cra_exit(struct crypto_tfm *tfm) { struct mtk_sha_ctx *tctx = crypto_tfm_ctx(tfm); if (tctx->flags & SHA_FLAGS_HMAC) { struct mtk_sha_hmac_ctx *bctx = tctx->base; crypto_free_shash(bctx->shash); } } static struct ahash_alg algs_sha1_sha224_sha256[] = { { .init = mtk_sha_init, .update = mtk_sha_update, .final = mtk_sha_final, .finup = mtk_sha_finup, .digest = mtk_sha_digest, .export = mtk_sha_export, .import = mtk_sha_import, .halg.digestsize = SHA1_DIGEST_SIZE, .halg.statesize = sizeof(struct mtk_sha_reqctx), .halg.base = { .cra_name = "sha1", .cra_driver_name = "mtk-sha1", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct mtk_sha_ctx), .cra_alignmask = SHA_ALIGN_MSK, .cra_module = THIS_MODULE, .cra_init = mtk_sha_cra_init, .cra_exit = mtk_sha_cra_exit, } }, { .init = mtk_sha_init, .update = mtk_sha_update, .final = mtk_sha_final, .finup = mtk_sha_finup, .digest = mtk_sha_digest, .export = mtk_sha_export, .import = mtk_sha_import, .halg.digestsize = SHA224_DIGEST_SIZE, .halg.statesize = sizeof(struct mtk_sha_reqctx), .halg.base = { .cra_name = "sha224", .cra_driver_name = "mtk-sha224", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct mtk_sha_ctx), .cra_alignmask = SHA_ALIGN_MSK, .cra_module = THIS_MODULE, .cra_init = mtk_sha_cra_init, .cra_exit = mtk_sha_cra_exit, } }, { .init = mtk_sha_init, .update = mtk_sha_update, .final = mtk_sha_final, .finup = mtk_sha_finup, .digest = mtk_sha_digest, .export = mtk_sha_export, .import = mtk_sha_import, .halg.digestsize = SHA256_DIGEST_SIZE, .halg.statesize = sizeof(struct mtk_sha_reqctx), .halg.base = { .cra_name = "sha256", .cra_driver_name = "mtk-sha256", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct mtk_sha_ctx), .cra_alignmask = SHA_ALIGN_MSK, .cra_module = THIS_MODULE, .cra_init = mtk_sha_cra_init, .cra_exit = mtk_sha_cra_exit, } }, { .init = mtk_sha_init, .update = mtk_sha_update, .final = mtk_sha_final, .finup = mtk_sha_finup, .digest = mtk_sha_digest, .export = mtk_sha_export, .import = mtk_sha_import, .setkey = mtk_sha_setkey, .halg.digestsize = SHA1_DIGEST_SIZE, .halg.statesize = sizeof(struct mtk_sha_reqctx), .halg.base = { .cra_name = "hmac(sha1)", .cra_driver_name = "mtk-hmac-sha1", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct mtk_sha_ctx) + sizeof(struct mtk_sha_hmac_ctx), .cra_alignmask = SHA_ALIGN_MSK, .cra_module = THIS_MODULE, .cra_init = mtk_sha_cra_sha1_init, .cra_exit = mtk_sha_cra_exit, } }, { .init = mtk_sha_init, .update = mtk_sha_update, .final = mtk_sha_final, .finup = mtk_sha_finup, .digest = mtk_sha_digest, .export = mtk_sha_export, .import = mtk_sha_import, .setkey = mtk_sha_setkey, .halg.digestsize = SHA224_DIGEST_SIZE, .halg.statesize = sizeof(struct mtk_sha_reqctx), .halg.base = { .cra_name = "hmac(sha224)", .cra_driver_name = "mtk-hmac-sha224", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct mtk_sha_ctx) + sizeof(struct mtk_sha_hmac_ctx), .cra_alignmask = SHA_ALIGN_MSK, .cra_module = THIS_MODULE, .cra_init = mtk_sha_cra_sha224_init, .cra_exit = mtk_sha_cra_exit, } }, { .init = mtk_sha_init, .update = mtk_sha_update, .final = mtk_sha_final, .finup = mtk_sha_finup, .digest = mtk_sha_digest, .export = mtk_sha_export, .import = mtk_sha_import, .setkey = mtk_sha_setkey, .halg.digestsize = SHA256_DIGEST_SIZE, .halg.statesize = sizeof(struct mtk_sha_reqctx), .halg.base = { .cra_name = "hmac(sha256)", .cra_driver_name = "mtk-hmac-sha256", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct mtk_sha_ctx) + sizeof(struct mtk_sha_hmac_ctx), .cra_alignmask = SHA_ALIGN_MSK, .cra_module = THIS_MODULE, .cra_init = mtk_sha_cra_sha256_init, .cra_exit = mtk_sha_cra_exit, } }, }; static struct ahash_alg algs_sha384_sha512[] = { { .init = mtk_sha_init, .update = mtk_sha_update, .final = mtk_sha_final, .finup = mtk_sha_finup, .digest = mtk_sha_digest, .export = mtk_sha_export, .import = mtk_sha_import, .halg.digestsize = SHA384_DIGEST_SIZE, .halg.statesize = sizeof(struct mtk_sha_reqctx), .halg.base = { .cra_name = "sha384", .cra_driver_name = "mtk-sha384", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct mtk_sha_ctx), .cra_alignmask = SHA_ALIGN_MSK, .cra_module = THIS_MODULE, .cra_init = mtk_sha_cra_init, .cra_exit = mtk_sha_cra_exit, } }, { .init = mtk_sha_init, .update = mtk_sha_update, .final = mtk_sha_final, .finup = mtk_sha_finup, .digest = mtk_sha_digest, .export = mtk_sha_export, .import = mtk_sha_import, .halg.digestsize = SHA512_DIGEST_SIZE, .halg.statesize = sizeof(struct mtk_sha_reqctx), .halg.base = { .cra_name = "sha512", .cra_driver_name = "mtk-sha512", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct mtk_sha_ctx), .cra_alignmask = SHA_ALIGN_MSK, .cra_module = THIS_MODULE, .cra_init = mtk_sha_cra_init, .cra_exit = mtk_sha_cra_exit, } }, { .init = mtk_sha_init, .update = mtk_sha_update, .final = mtk_sha_final, .finup = mtk_sha_finup, .digest = mtk_sha_digest, .export = mtk_sha_export, .import = mtk_sha_import, .setkey = mtk_sha_setkey, .halg.digestsize = SHA384_DIGEST_SIZE, .halg.statesize = sizeof(struct mtk_sha_reqctx), .halg.base = { .cra_name = "hmac(sha384)", .cra_driver_name = "mtk-hmac-sha384", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct mtk_sha_ctx) + sizeof(struct mtk_sha_hmac_ctx), .cra_alignmask = SHA_ALIGN_MSK, .cra_module = THIS_MODULE, .cra_init = mtk_sha_cra_sha384_init, .cra_exit = mtk_sha_cra_exit, } }, { .init = mtk_sha_init, .update = mtk_sha_update, .final = mtk_sha_final, .finup = mtk_sha_finup, .digest = mtk_sha_digest, .export = mtk_sha_export, .import = mtk_sha_import, .setkey = mtk_sha_setkey, .halg.digestsize = SHA512_DIGEST_SIZE, .halg.statesize = sizeof(struct mtk_sha_reqctx), .halg.base = { .cra_name = "hmac(sha512)", .cra_driver_name = "mtk-hmac-sha512", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct mtk_sha_ctx) + sizeof(struct mtk_sha_hmac_ctx), .cra_alignmask = SHA_ALIGN_MSK, .cra_module = THIS_MODULE, .cra_init = mtk_sha_cra_sha512_init, .cra_exit = mtk_sha_cra_exit, } }, }; static void mtk_sha_queue_task(unsigned long data) { struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; mtk_sha_handle_queue(sha->cryp, sha->id - MTK_RING2, NULL); } static void mtk_sha_done_task(unsigned long data) { struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; struct mtk_cryp *cryp = sha->cryp; mtk_sha_unmap(cryp, sha); mtk_sha_complete(cryp, sha); } static irqreturn_t mtk_sha_irq(int irq, void *dev_id) { struct mtk_sha_rec *sha = (struct mtk_sha_rec *)dev_id; struct mtk_cryp *cryp = sha->cryp; u32 val = mtk_sha_read(cryp, RDR_STAT(sha->id)); mtk_sha_write(cryp, RDR_STAT(sha->id), val); if (likely((SHA_FLAGS_BUSY & sha->flags))) { mtk_sha_write(cryp, RDR_PROC_COUNT(sha->id), MTK_CNT_RST); mtk_sha_write(cryp, RDR_THRESH(sha->id), MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); tasklet_schedule(&sha->done_task); } else { dev_warn(cryp->dev, "SHA interrupt when no active requests.\n"); } return IRQ_HANDLED; } /* * The purpose of two SHA records is used to get extra performance. * It is similar to mtk_aes_record_init(). */ static int mtk_sha_record_init(struct mtk_cryp *cryp) { struct mtk_sha_rec **sha = cryp->sha; int i, err = -ENOMEM; for (i = 0; i < MTK_REC_NUM; i++) { sha[i] = kzalloc(sizeof(**sha), GFP_KERNEL); if (!sha[i]) goto err_cleanup; sha[i]->cryp = cryp; spin_lock_init(&sha[i]->lock); crypto_init_queue(&sha[i]->queue, SHA_QUEUE_SIZE); tasklet_init(&sha[i]->queue_task, mtk_sha_queue_task, (unsigned long)sha[i]); tasklet_init(&sha[i]->done_task, mtk_sha_done_task, (unsigned long)sha[i]); } /* Link to ring2 and ring3 respectively */ sha[0]->id = MTK_RING2; sha[1]->id = MTK_RING3; cryp->rec = 1; return 0; err_cleanup: for (; i--; ) kfree(sha[i]); return err; } static void mtk_sha_record_free(struct mtk_cryp *cryp) { int i; for (i = 0; i < MTK_REC_NUM; i++) { tasklet_kill(&cryp->sha[i]->done_task); tasklet_kill(&cryp->sha[i]->queue_task); kfree(cryp->sha[i]); } } static void mtk_sha_unregister_algs(void) { int i; for (i = 0; i < ARRAY_SIZE(algs_sha1_sha224_sha256); i++) crypto_unregister_ahash(&algs_sha1_sha224_sha256[i]); for (i = 0; i < ARRAY_SIZE(algs_sha384_sha512); i++) crypto_unregister_ahash(&algs_sha384_sha512[i]); } static int mtk_sha_register_algs(void) { int err, i; for (i = 0; i < ARRAY_SIZE(algs_sha1_sha224_sha256); i++) { err = crypto_register_ahash(&algs_sha1_sha224_sha256[i]); if (err) goto err_sha_224_256_algs; } for (i = 0; i < ARRAY_SIZE(algs_sha384_sha512); i++) { err = crypto_register_ahash(&algs_sha384_sha512[i]); if (err) goto err_sha_384_512_algs; } return 0; err_sha_384_512_algs: for (; i--; ) crypto_unregister_ahash(&algs_sha384_sha512[i]); i = ARRAY_SIZE(algs_sha1_sha224_sha256); err_sha_224_256_algs: for (; i--; ) crypto_unregister_ahash(&algs_sha1_sha224_sha256[i]); return err; } int mtk_hash_alg_register(struct mtk_cryp *cryp) { int err; INIT_LIST_HEAD(&cryp->sha_list); /* Initialize two hash records */ err = mtk_sha_record_init(cryp); if (err) goto err_record; err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING2], mtk_sha_irq, 0, "mtk-sha", cryp->sha[0]); if (err) { dev_err(cryp->dev, "unable to request sha irq0.\n"); goto err_res; } err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING3], mtk_sha_irq, 0, "mtk-sha", cryp->sha[1]); if (err) { dev_err(cryp->dev, "unable to request sha irq1.\n"); goto err_res; } /* Enable ring2 and ring3 interrupt for hash */ mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING2), MTK_IRQ_RDR2); mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING3), MTK_IRQ_RDR3); spin_lock(&mtk_sha.lock); list_add_tail(&cryp->sha_list, &mtk_sha.dev_list); spin_unlock(&mtk_sha.lock); err = mtk_sha_register_algs(); if (err) goto err_algs; return 0; err_algs: spin_lock(&mtk_sha.lock); list_del(&cryp->sha_list); spin_unlock(&mtk_sha.lock); err_res: mtk_sha_record_free(cryp); err_record: dev_err(cryp->dev, "mtk-sha initialization failed.\n"); return err; } void mtk_hash_alg_release(struct mtk_cryp *cryp) { spin_lock(&mtk_sha.lock); list_del(&cryp->sha_list); spin_unlock(&mtk_sha.lock); mtk_sha_unregister_algs(); mtk_sha_record_free(cryp); }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1