v4.19.13 snapshot.
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
new file mode 100644
index 0000000..5cf6474
--- /dev/null
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
@@ -0,0 +1,546 @@
+/*
+ * sun4i-ss-cipher.c - hardware cryptographic accelerator for Allwinner A20 SoC
+ *
+ * Copyright (C) 2013-2015 Corentin LABBE <clabbe.montjoie@gmail.com>
+ *
+ * This file add support for AES cipher with 128,192,256 bits
+ * keysize in CBC and ECB mode.
+ * Add support also for DES and 3DES in CBC and ECB mode.
+ *
+ * You could find the datasheet in Documentation/arm/sunxi/README
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+#include "sun4i-ss.h"
+
+static int sun4i_ss_opti_poll(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_ss_ctx *ss = op->ss;
+ unsigned int ivsize = crypto_skcipher_ivsize(tfm);
+ struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq);
+ u32 mode = ctx->mode;
+ /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */
+ u32 rx_cnt = SS_RX_DEFAULT;
+ u32 tx_cnt = 0;
+ u32 spaces;
+ u32 v;
+ int err = 0;
+ unsigned int i;
+ unsigned int ileft = areq->cryptlen;
+ unsigned int oleft = areq->cryptlen;
+ unsigned int todo;
+ struct sg_mapping_iter mi, mo;
+ unsigned int oi, oo; /* offset for in and out */
+ unsigned long flags;
+
+ if (!areq->cryptlen)
+ return 0;
+
+ if (!areq->iv) {
+ dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n");
+ return -EINVAL;
+ }
+
+ if (!areq->src || !areq->dst) {
+ dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n");
+ return -EINVAL;
+ }
+
+ spin_lock_irqsave(&ss->slock, flags);
+
+ for (i = 0; i < op->keylen; i += 4)
+ writel(*(op->key + i / 4), ss->base + SS_KEY0 + i);
+
+ if (areq->iv) {
+ for (i = 0; i < 4 && i < ivsize / 4; i++) {
+ v = *(u32 *)(areq->iv + i * 4);
+ writel(v, ss->base + SS_IV0 + i * 4);
+ }
+ }
+ writel(mode, ss->base + SS_CTL);
+
+ sg_miter_start(&mi, areq->src, sg_nents(areq->src),
+ SG_MITER_FROM_SG | SG_MITER_ATOMIC);
+ sg_miter_start(&mo, areq->dst, sg_nents(areq->dst),
+ SG_MITER_TO_SG | SG_MITER_ATOMIC);
+ sg_miter_next(&mi);
+ sg_miter_next(&mo);
+ if (!mi.addr || !mo.addr) {
+ dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");
+ err = -EINVAL;
+ goto release_ss;
+ }
+
+ ileft = areq->cryptlen / 4;
+ oleft = areq->cryptlen / 4;
+ oi = 0;
+ oo = 0;
+ do {
+ todo = min3(rx_cnt, ileft, (mi.length - oi) / 4);
+ if (todo) {
+ ileft -= todo;
+ writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo);
+ oi += todo * 4;
+ }
+ if (oi == mi.length) {
+ sg_miter_next(&mi);
+ oi = 0;
+ }
+
+ spaces = readl(ss->base + SS_FCSR);
+ rx_cnt = SS_RXFIFO_SPACES(spaces);
+ tx_cnt = SS_TXFIFO_SPACES(spaces);
+
+ todo = min3(tx_cnt, oleft, (mo.length - oo) / 4);
+ if (todo) {
+ oleft -= todo;
+ readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo);
+ oo += todo * 4;
+ }
+ if (oo == mo.length) {
+ sg_miter_next(&mo);
+ oo = 0;
+ }
+ } while (oleft);
+
+ if (areq->iv) {
+ for (i = 0; i < 4 && i < ivsize / 4; i++) {
+ v = readl(ss->base + SS_IV0 + i * 4);
+ *(u32 *)(areq->iv + i * 4) = v;
+ }
+ }
+
+release_ss:
+ sg_miter_stop(&mi);
+ sg_miter_stop(&mo);
+ writel(0, ss->base + SS_CTL);
+ spin_unlock_irqrestore(&ss->slock, flags);
+ return err;
+}
+
+/* Generic function that support SG with size not multiple of 4 */
+static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_ss_ctx *ss = op->ss;
+ int no_chunk = 1;
+ struct scatterlist *in_sg = areq->src;
+ struct scatterlist *out_sg = areq->dst;
+ unsigned int ivsize = crypto_skcipher_ivsize(tfm);
+ struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq);
+ u32 mode = ctx->mode;
+ /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */
+ u32 rx_cnt = SS_RX_DEFAULT;
+ u32 tx_cnt = 0;
+ u32 v;
+ u32 spaces;
+ int err = 0;
+ unsigned int i;
+ unsigned int ileft = areq->cryptlen;
+ unsigned int oleft = areq->cryptlen;
+ unsigned int todo;
+ struct sg_mapping_iter mi, mo;
+ unsigned int oi, oo; /* offset for in and out */
+ char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */
+ char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */
+ unsigned int ob = 0; /* offset in buf */
+ unsigned int obo = 0; /* offset in bufo*/
+ unsigned int obl = 0; /* length of data in bufo */
+ unsigned long flags;
+
+ if (!areq->cryptlen)
+ return 0;
+
+ if (!areq->iv) {
+ dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n");
+ return -EINVAL;
+ }
+
+ if (!areq->src || !areq->dst) {
+ dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n");
+ return -EINVAL;
+ }
+
+ /*
+ * if we have only SGs with size multiple of 4,
+ * we can use the SS optimized function
+ */
+ while (in_sg && no_chunk == 1) {
+ if (in_sg->length % 4)
+ no_chunk = 0;
+ in_sg = sg_next(in_sg);
+ }
+ while (out_sg && no_chunk == 1) {
+ if (out_sg->length % 4)
+ no_chunk = 0;
+ out_sg = sg_next(out_sg);
+ }
+
+ if (no_chunk == 1)
+ return sun4i_ss_opti_poll(areq);
+
+ spin_lock_irqsave(&ss->slock, flags);
+
+ for (i = 0; i < op->keylen; i += 4)
+ writel(*(op->key + i / 4), ss->base + SS_KEY0 + i);
+
+ if (areq->iv) {
+ for (i = 0; i < 4 && i < ivsize / 4; i++) {
+ v = *(u32 *)(areq->iv + i * 4);
+ writel(v, ss->base + SS_IV0 + i * 4);
+ }
+ }
+ writel(mode, ss->base + SS_CTL);
+
+ sg_miter_start(&mi, areq->src, sg_nents(areq->src),
+ SG_MITER_FROM_SG | SG_MITER_ATOMIC);
+ sg_miter_start(&mo, areq->dst, sg_nents(areq->dst),
+ SG_MITER_TO_SG | SG_MITER_ATOMIC);
+ sg_miter_next(&mi);
+ sg_miter_next(&mo);
+ if (!mi.addr || !mo.addr) {
+ dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");
+ err = -EINVAL;
+ goto release_ss;
+ }
+ ileft = areq->cryptlen;
+ oleft = areq->cryptlen;
+ oi = 0;
+ oo = 0;
+
+ while (oleft) {
+ if (ileft) {
+ /*
+ * todo is the number of consecutive 4byte word that we
+ * can read from current SG
+ */
+ todo = min3(rx_cnt, ileft / 4, (mi.length - oi) / 4);
+ if (todo && !ob) {
+ writesl(ss->base + SS_RXFIFO, mi.addr + oi,
+ todo);
+ ileft -= todo * 4;
+ oi += todo * 4;
+ } else {
+ /*
+ * not enough consecutive bytes, so we need to
+ * linearize in buf. todo is in bytes
+ * After that copy, if we have a multiple of 4
+ * we need to be able to write all buf in one
+ * pass, so it is why we min() with rx_cnt
+ */
+ todo = min3(rx_cnt * 4 - ob, ileft,
+ mi.length - oi);
+ memcpy(buf + ob, mi.addr + oi, todo);
+ ileft -= todo;
+ oi += todo;
+ ob += todo;
+ if (!(ob % 4)) {
+ writesl(ss->base + SS_RXFIFO, buf,
+ ob / 4);
+ ob = 0;
+ }
+ }
+ if (oi == mi.length) {
+ sg_miter_next(&mi);
+ oi = 0;
+ }
+ }
+
+ spaces = readl(ss->base + SS_FCSR);
+ rx_cnt = SS_RXFIFO_SPACES(spaces);
+ tx_cnt = SS_TXFIFO_SPACES(spaces);
+ dev_dbg(ss->dev, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u\n",
+ mode,
+ oi, mi.length, ileft, areq->cryptlen, rx_cnt,
+ oo, mo.length, oleft, areq->cryptlen, tx_cnt, ob);
+
+ if (!tx_cnt)
+ continue;
+ /* todo in 4bytes word */
+ todo = min3(tx_cnt, oleft / 4, (mo.length - oo) / 4);
+ if (todo) {
+ readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo);
+ oleft -= todo * 4;
+ oo += todo * 4;
+ if (oo == mo.length) {
+ sg_miter_next(&mo);
+ oo = 0;
+ }
+ } else {
+ /*
+ * read obl bytes in bufo, we read at maximum for
+ * emptying the device
+ */
+ readsl(ss->base + SS_TXFIFO, bufo, tx_cnt);
+ obl = tx_cnt * 4;
+ obo = 0;
+ do {
+ /*
+ * how many bytes we can copy ?
+ * no more than remaining SG size
+ * no more than remaining buffer
+ * no need to test against oleft
+ */
+ todo = min(mo.length - oo, obl - obo);
+ memcpy(mo.addr + oo, bufo + obo, todo);
+ oleft -= todo;
+ obo += todo;
+ oo += todo;
+ if (oo == mo.length) {
+ sg_miter_next(&mo);
+ oo = 0;
+ }
+ } while (obo < obl);
+ /* bufo must be fully used here */
+ }
+ }
+ if (areq->iv) {
+ for (i = 0; i < 4 && i < ivsize / 4; i++) {
+ v = readl(ss->base + SS_IV0 + i * 4);
+ *(u32 *)(areq->iv + i * 4) = v;
+ }
+ }
+
+release_ss:
+ sg_miter_stop(&mi);
+ sg_miter_stop(&mo);
+ writel(0, ss->base + SS_CTL);
+ spin_unlock_irqrestore(&ss->slock, flags);
+
+ return err;
+}
+
+/* CBC AES */
+int sun4i_ss_cbc_aes_encrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+int sun4i_ss_cbc_aes_decrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_DECRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+/* ECB AES */
+int sun4i_ss_ecb_aes_encrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+int sun4i_ss_ecb_aes_decrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_DECRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+/* CBC DES */
+int sun4i_ss_cbc_des_encrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+int sun4i_ss_cbc_des_decrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_DECRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+/* ECB DES */
+int sun4i_ss_ecb_des_encrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+int sun4i_ss_ecb_des_decrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_DECRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+/* CBC 3DES */
+int sun4i_ss_cbc_des3_encrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+int sun4i_ss_cbc_des3_decrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_DECRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+/* ECB 3DES */
+int sun4i_ss_ecb_des3_encrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+int sun4i_ss_ecb_des3_decrypt(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+
+ rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_DECRYPTION |
+ op->keymode;
+ return sun4i_ss_cipher_poll(areq);
+}
+
+int sun4i_ss_cipher_init(struct crypto_tfm *tfm)
+{
+ struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);
+ struct sun4i_ss_alg_template *algt;
+
+ memset(op, 0, sizeof(struct sun4i_tfm_ctx));
+
+ algt = container_of(tfm->__crt_alg, struct sun4i_ss_alg_template,
+ alg.crypto.base);
+ op->ss = algt->ss;
+
+ crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
+ sizeof(struct sun4i_cipher_req_ctx));
+
+ return 0;
+}
+
+/* check and set the AES key, prepare the mode to be used */
+int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_ss_ctx *ss = op->ss;
+
+ switch (keylen) {
+ case 128 / 8:
+ op->keymode = SS_AES_128BITS;
+ break;
+ case 192 / 8:
+ op->keymode = SS_AES_192BITS;
+ break;
+ case 256 / 8:
+ op->keymode = SS_AES_256BITS;
+ break;
+ default:
+ dev_err(ss->dev, "ERROR: Invalid keylen %u\n", keylen);
+ crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+ op->keylen = keylen;
+ memcpy(op->key, key, keylen);
+ return 0;
+}
+
+/* check and set the DES key, prepare the mode to be used */
+int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_ss_ctx *ss = op->ss;
+ u32 flags;
+ u32 tmp[DES_EXPKEY_WORDS];
+ int ret;
+
+ if (unlikely(keylen != DES_KEY_SIZE)) {
+ dev_err(ss->dev, "Invalid keylen %u\n", keylen);
+ crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
+ flags = crypto_skcipher_get_flags(tfm);
+
+ ret = des_ekey(tmp, key);
+ if (unlikely(!ret) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
+ crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY);
+ dev_dbg(ss->dev, "Weak key %u\n", keylen);
+ return -EINVAL;
+ }
+
+ op->keylen = keylen;
+ memcpy(op->key, key, keylen);
+ return 0;
+}
+
+/* check and set the 3DES key, prepare the mode to be used */
+int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_ss_ctx *ss = op->ss;
+
+ if (unlikely(keylen != 3 * DES_KEY_SIZE)) {
+ dev_err(ss->dev, "Invalid keylen %u\n", keylen);
+ crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+ op->keylen = keylen;
+ memcpy(op->key, key, keylen);
+ return 0;
+}