Update Linux to v5.4.2
Change-Id: Idf6911045d9d382da2cfe01b1edff026404ac8fd
diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile
index a51010e..12889d4 100644
--- a/arch/s390/crypto/Makefile
+++ b/arch/s390/crypto/Makefile
@@ -6,6 +6,8 @@
obj-$(CONFIG_CRYPTO_SHA1_S390) += sha1_s390.o sha_common.o
obj-$(CONFIG_CRYPTO_SHA256_S390) += sha256_s390.o sha_common.o
obj-$(CONFIG_CRYPTO_SHA512_S390) += sha512_s390.o sha_common.o
+obj-$(CONFIG_CRYPTO_SHA3_256_S390) += sha3_256_s390.o sha_common.o
+obj-$(CONFIG_CRYPTO_SHA3_512_S390) += sha3_512_s390.o sha_common.o
obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o
obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o
obj-$(CONFIG_CRYPTO_PAES_S390) += paes_s390.o
diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c
index c54cb26..9803e96 100644
--- a/arch/s390/crypto/aes_s390.c
+++ b/arch/s390/crypto/aes_s390.c
@@ -27,14 +27,14 @@
#include <linux/module.h>
#include <linux/cpufeature.h>
#include <linux/init.h>
-#include <linux/spinlock.h>
+#include <linux/mutex.h>
#include <linux/fips.h>
#include <linux/string.h>
#include <crypto/xts.h>
#include <asm/cpacf.h>
static u8 *ctrblk;
-static DEFINE_SPINLOCK(ctrblk_lock);
+static DEFINE_MUTEX(ctrblk_lock);
static cpacf_mask_t km_functions, kmc_functions, kmctr_functions,
kma_functions;
@@ -44,7 +44,7 @@
int key_len;
unsigned long fc;
union {
- struct crypto_skcipher *blk;
+ struct crypto_sync_skcipher *blk;
struct crypto_cipher *cip;
} fallback;
};
@@ -54,7 +54,7 @@
u8 pcc_key[32];
int key_len;
unsigned long fc;
- struct crypto_skcipher *fallback;
+ struct crypto_sync_skcipher *fallback;
};
struct gcm_sg_walk {
@@ -108,7 +108,7 @@
return 0;
}
-static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
@@ -119,7 +119,7 @@
cpacf_km(sctx->fc, &sctx->key, out, in, AES_BLOCK_SIZE);
}
-static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
@@ -137,7 +137,7 @@
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
sctx->fallback.cip = crypto_alloc_cipher(name, 0,
- CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+ CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(sctx->fallback.cip)) {
pr_err("Allocating AES fallback algorithm %s failed\n",
@@ -172,8 +172,8 @@
.cia_min_keysize = AES_MIN_KEY_SIZE,
.cia_max_keysize = AES_MAX_KEY_SIZE,
.cia_setkey = aes_set_key,
- .cia_encrypt = aes_encrypt,
- .cia_decrypt = aes_decrypt,
+ .cia_encrypt = crypto_aes_encrypt,
+ .cia_decrypt = crypto_aes_decrypt,
}
}
};
@@ -184,14 +184,15 @@
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
unsigned int ret;
- crypto_skcipher_clear_flags(sctx->fallback.blk, CRYPTO_TFM_REQ_MASK);
- crypto_skcipher_set_flags(sctx->fallback.blk, tfm->crt_flags &
+ crypto_sync_skcipher_clear_flags(sctx->fallback.blk,
+ CRYPTO_TFM_REQ_MASK);
+ crypto_sync_skcipher_set_flags(sctx->fallback.blk, tfm->crt_flags &
CRYPTO_TFM_REQ_MASK);
- ret = crypto_skcipher_setkey(sctx->fallback.blk, key, len);
+ ret = crypto_sync_skcipher_setkey(sctx->fallback.blk, key, len);
tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
- tfm->crt_flags |= crypto_skcipher_get_flags(sctx->fallback.blk) &
+ tfm->crt_flags |= crypto_sync_skcipher_get_flags(sctx->fallback.blk) &
CRYPTO_TFM_RES_MASK;
return ret;
@@ -204,9 +205,9 @@
unsigned int ret;
struct crypto_blkcipher *tfm = desc->tfm;
struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(tfm);
- SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk);
+ SYNC_SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk);
- skcipher_request_set_tfm(req, sctx->fallback.blk);
+ skcipher_request_set_sync_tfm(req, sctx->fallback.blk);
skcipher_request_set_callback(req, desc->flags, NULL, NULL);
skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
@@ -223,9 +224,9 @@
unsigned int ret;
struct crypto_blkcipher *tfm = desc->tfm;
struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(tfm);
- SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk);
+ SYNC_SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk);
- skcipher_request_set_tfm(req, sctx->fallback.blk);
+ skcipher_request_set_sync_tfm(req, sctx->fallback.blk);
skcipher_request_set_callback(req, desc->flags, NULL, NULL);
skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
@@ -306,8 +307,7 @@
const char *name = tfm->__crt_alg->cra_name;
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
- sctx->fallback.blk = crypto_alloc_skcipher(name, 0,
- CRYPTO_ALG_ASYNC |
+ sctx->fallback.blk = crypto_alloc_sync_skcipher(name, 0,
CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(sctx->fallback.blk)) {
@@ -323,7 +323,7 @@
{
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
- crypto_free_skcipher(sctx->fallback.blk);
+ crypto_free_sync_skcipher(sctx->fallback.blk);
}
static struct crypto_alg ecb_aes_alg = {
@@ -453,14 +453,15 @@
struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
unsigned int ret;
- crypto_skcipher_clear_flags(xts_ctx->fallback, CRYPTO_TFM_REQ_MASK);
- crypto_skcipher_set_flags(xts_ctx->fallback, tfm->crt_flags &
+ crypto_sync_skcipher_clear_flags(xts_ctx->fallback,
+ CRYPTO_TFM_REQ_MASK);
+ crypto_sync_skcipher_set_flags(xts_ctx->fallback, tfm->crt_flags &
CRYPTO_TFM_REQ_MASK);
- ret = crypto_skcipher_setkey(xts_ctx->fallback, key, len);
+ ret = crypto_sync_skcipher_setkey(xts_ctx->fallback, key, len);
tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
- tfm->crt_flags |= crypto_skcipher_get_flags(xts_ctx->fallback) &
+ tfm->crt_flags |= crypto_sync_skcipher_get_flags(xts_ctx->fallback) &
CRYPTO_TFM_RES_MASK;
return ret;
@@ -472,10 +473,10 @@
{
struct crypto_blkcipher *tfm = desc->tfm;
struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(tfm);
- SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback);
+ SYNC_SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback);
unsigned int ret;
- skcipher_request_set_tfm(req, xts_ctx->fallback);
+ skcipher_request_set_sync_tfm(req, xts_ctx->fallback);
skcipher_request_set_callback(req, desc->flags, NULL, NULL);
skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
@@ -491,10 +492,10 @@
{
struct crypto_blkcipher *tfm = desc->tfm;
struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(tfm);
- SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback);
+ SYNC_SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback);
unsigned int ret;
- skcipher_request_set_tfm(req, xts_ctx->fallback);
+ skcipher_request_set_sync_tfm(req, xts_ctx->fallback);
skcipher_request_set_callback(req, desc->flags, NULL, NULL);
skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
@@ -511,7 +512,7 @@
unsigned long fc;
int err;
- err = xts_check_key(tfm, in_key, key_len);
+ err = xts_fallback_setkey(tfm, in_key, key_len);
if (err)
return err;
@@ -528,7 +529,7 @@
/* Check if the function code is available */
xts_ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
if (!xts_ctx->fc)
- return xts_fallback_setkey(tfm, in_key, key_len);
+ return 0;
/* Split the XTS key into the two subkeys */
key_len = key_len / 2;
@@ -585,7 +586,10 @@
struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk walk;
- if (unlikely(!xts_ctx->fc))
+ if (!nbytes)
+ return -EINVAL;
+
+ if (unlikely(!xts_ctx->fc || (nbytes % XTS_BLOCK_SIZE) != 0))
return xts_fallback_encrypt(desc, dst, src, nbytes);
blkcipher_walk_init(&walk, dst, src, nbytes);
@@ -599,7 +603,10 @@
struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk walk;
- if (unlikely(!xts_ctx->fc))
+ if (!nbytes)
+ return -EINVAL;
+
+ if (unlikely(!xts_ctx->fc || (nbytes % XTS_BLOCK_SIZE) != 0))
return xts_fallback_decrypt(desc, dst, src, nbytes);
blkcipher_walk_init(&walk, dst, src, nbytes);
@@ -611,8 +618,7 @@
const char *name = tfm->__crt_alg->cra_name;
struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
- xts_ctx->fallback = crypto_alloc_skcipher(name, 0,
- CRYPTO_ALG_ASYNC |
+ xts_ctx->fallback = crypto_alloc_sync_skcipher(name, 0,
CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(xts_ctx->fallback)) {
@@ -627,7 +633,7 @@
{
struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
- crypto_free_skcipher(xts_ctx->fallback);
+ crypto_free_sync_skcipher(xts_ctx->fallback);
}
static struct crypto_alg xts_aes_alg = {
@@ -698,7 +704,7 @@
unsigned int n, nbytes;
int ret, locked;
- locked = spin_trylock(&ctrblk_lock);
+ locked = mutex_trylock(&ctrblk_lock);
ret = blkcipher_walk_virt_block(desc, walk, AES_BLOCK_SIZE);
while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) {
@@ -716,7 +722,7 @@
ret = blkcipher_walk_done(desc, walk, nbytes - n);
}
if (locked)
- spin_unlock(&ctrblk_lock);
+ mutex_unlock(&ctrblk_lock);
/*
* final block may be < AES_BLOCK_SIZE, copy only nbytes
*/
@@ -826,19 +832,45 @@
return 0;
}
-static void gcm_sg_walk_start(struct gcm_sg_walk *gw, struct scatterlist *sg,
- unsigned int len)
+static void gcm_walk_start(struct gcm_sg_walk *gw, struct scatterlist *sg,
+ unsigned int len)
{
memset(gw, 0, sizeof(*gw));
gw->walk_bytes_remain = len;
scatterwalk_start(&gw->walk, sg);
}
-static int gcm_sg_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded)
+static inline unsigned int _gcm_sg_clamp_and_map(struct gcm_sg_walk *gw)
+{
+ struct scatterlist *nextsg;
+
+ gw->walk_bytes = scatterwalk_clamp(&gw->walk, gw->walk_bytes_remain);
+ while (!gw->walk_bytes) {
+ nextsg = sg_next(gw->walk.sg);
+ if (!nextsg)
+ return 0;
+ scatterwalk_start(&gw->walk, nextsg);
+ gw->walk_bytes = scatterwalk_clamp(&gw->walk,
+ gw->walk_bytes_remain);
+ }
+ gw->walk_ptr = scatterwalk_map(&gw->walk);
+ return gw->walk_bytes;
+}
+
+static inline void _gcm_sg_unmap_and_advance(struct gcm_sg_walk *gw,
+ unsigned int nbytes)
+{
+ gw->walk_bytes_remain -= nbytes;
+ scatterwalk_unmap(&gw->walk);
+ scatterwalk_advance(&gw->walk, nbytes);
+ scatterwalk_done(&gw->walk, 0, gw->walk_bytes_remain);
+ gw->walk_ptr = NULL;
+}
+
+static int gcm_in_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded)
{
int n;
- /* minbytesneeded <= AES_BLOCK_SIZE */
if (gw->buf_bytes && gw->buf_bytes >= minbytesneeded) {
gw->ptr = gw->buf;
gw->nbytes = gw->buf_bytes;
@@ -851,13 +883,11 @@
goto out;
}
- gw->walk_bytes = scatterwalk_clamp(&gw->walk, gw->walk_bytes_remain);
- if (!gw->walk_bytes) {
- scatterwalk_start(&gw->walk, sg_next(gw->walk.sg));
- gw->walk_bytes = scatterwalk_clamp(&gw->walk,
- gw->walk_bytes_remain);
+ if (!_gcm_sg_clamp_and_map(gw)) {
+ gw->ptr = NULL;
+ gw->nbytes = 0;
+ goto out;
}
- gw->walk_ptr = scatterwalk_map(&gw->walk);
if (!gw->buf_bytes && gw->walk_bytes >= minbytesneeded) {
gw->ptr = gw->walk_ptr;
@@ -869,51 +899,90 @@
n = min(gw->walk_bytes, AES_BLOCK_SIZE - gw->buf_bytes);
memcpy(gw->buf + gw->buf_bytes, gw->walk_ptr, n);
gw->buf_bytes += n;
- gw->walk_bytes_remain -= n;
- scatterwalk_unmap(&gw->walk);
- scatterwalk_advance(&gw->walk, n);
- scatterwalk_done(&gw->walk, 0, gw->walk_bytes_remain);
-
+ _gcm_sg_unmap_and_advance(gw, n);
if (gw->buf_bytes >= minbytesneeded) {
gw->ptr = gw->buf;
gw->nbytes = gw->buf_bytes;
goto out;
}
-
- gw->walk_bytes = scatterwalk_clamp(&gw->walk,
- gw->walk_bytes_remain);
- if (!gw->walk_bytes) {
- scatterwalk_start(&gw->walk, sg_next(gw->walk.sg));
- gw->walk_bytes = scatterwalk_clamp(&gw->walk,
- gw->walk_bytes_remain);
+ if (!_gcm_sg_clamp_and_map(gw)) {
+ gw->ptr = NULL;
+ gw->nbytes = 0;
+ goto out;
}
- gw->walk_ptr = scatterwalk_map(&gw->walk);
}
out:
return gw->nbytes;
}
-static void gcm_sg_walk_done(struct gcm_sg_walk *gw, unsigned int bytesdone)
+static int gcm_out_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded)
{
- int n;
+ if (gw->walk_bytes_remain == 0) {
+ gw->ptr = NULL;
+ gw->nbytes = 0;
+ goto out;
+ }
+ if (!_gcm_sg_clamp_and_map(gw)) {
+ gw->ptr = NULL;
+ gw->nbytes = 0;
+ goto out;
+ }
+
+ if (gw->walk_bytes >= minbytesneeded) {
+ gw->ptr = gw->walk_ptr;
+ gw->nbytes = gw->walk_bytes;
+ goto out;
+ }
+
+ scatterwalk_unmap(&gw->walk);
+ gw->walk_ptr = NULL;
+
+ gw->ptr = gw->buf;
+ gw->nbytes = sizeof(gw->buf);
+
+out:
+ return gw->nbytes;
+}
+
+static int gcm_in_walk_done(struct gcm_sg_walk *gw, unsigned int bytesdone)
+{
if (gw->ptr == NULL)
- return;
+ return 0;
if (gw->ptr == gw->buf) {
- n = gw->buf_bytes - bytesdone;
+ int n = gw->buf_bytes - bytesdone;
if (n > 0) {
memmove(gw->buf, gw->buf + bytesdone, n);
- gw->buf_bytes -= n;
+ gw->buf_bytes = n;
} else
gw->buf_bytes = 0;
- } else {
- gw->walk_bytes_remain -= bytesdone;
- scatterwalk_unmap(&gw->walk);
- scatterwalk_advance(&gw->walk, bytesdone);
- scatterwalk_done(&gw->walk, 0, gw->walk_bytes_remain);
- }
+ } else
+ _gcm_sg_unmap_and_advance(gw, bytesdone);
+
+ return bytesdone;
+}
+
+static int gcm_out_walk_done(struct gcm_sg_walk *gw, unsigned int bytesdone)
+{
+ int i, n;
+
+ if (gw->ptr == NULL)
+ return 0;
+
+ if (gw->ptr == gw->buf) {
+ for (i = 0; i < bytesdone; i += n) {
+ if (!_gcm_sg_clamp_and_map(gw))
+ return i;
+ n = min(gw->walk_bytes, bytesdone - i);
+ memcpy(gw->walk_ptr, gw->buf + i, n);
+ _gcm_sg_unmap_and_advance(gw, n);
+ }
+ } else
+ _gcm_sg_unmap_and_advance(gw, bytesdone);
+
+ return bytesdone;
}
static int gcm_aes_crypt(struct aead_request *req, unsigned int flags)
@@ -926,7 +995,7 @@
unsigned int pclen = req->cryptlen;
int ret = 0;
- unsigned int len, in_bytes, out_bytes,
+ unsigned int n, len, in_bytes, out_bytes,
min_bytes, bytes, aad_bytes, pc_bytes;
struct gcm_sg_walk gw_in, gw_out;
u8 tag[GHASH_DIGEST_SIZE];
@@ -963,14 +1032,14 @@
*(u32 *)(param.j0 + ivsize) = 1;
memcpy(param.k, ctx->key, ctx->key_len);
- gcm_sg_walk_start(&gw_in, req->src, len);
- gcm_sg_walk_start(&gw_out, req->dst, len);
+ gcm_walk_start(&gw_in, req->src, len);
+ gcm_walk_start(&gw_out, req->dst, len);
do {
min_bytes = min_t(unsigned int,
aadlen > 0 ? aadlen : pclen, AES_BLOCK_SIZE);
- in_bytes = gcm_sg_walk_go(&gw_in, min_bytes);
- out_bytes = gcm_sg_walk_go(&gw_out, min_bytes);
+ in_bytes = gcm_in_walk_go(&gw_in, min_bytes);
+ out_bytes = gcm_out_walk_go(&gw_out, min_bytes);
bytes = min(in_bytes, out_bytes);
if (aadlen + pclen <= bytes) {
@@ -997,8 +1066,11 @@
gw_in.ptr + aad_bytes, pc_bytes,
gw_in.ptr, aad_bytes);
- gcm_sg_walk_done(&gw_in, aad_bytes + pc_bytes);
- gcm_sg_walk_done(&gw_out, aad_bytes + pc_bytes);
+ n = aad_bytes + pc_bytes;
+ if (gcm_in_walk_done(&gw_in, n) != n)
+ return -ENOMEM;
+ if (gcm_out_walk_done(&gw_out, n) != n)
+ return -ENOMEM;
aadlen -= aad_bytes;
pclen -= pc_bytes;
} while (aadlen + pclen > 0);
diff --git a/arch/s390/crypto/crc32be-vx.S b/arch/s390/crypto/crc32be-vx.S
index 2bf01ba..0099044 100644
--- a/arch/s390/crypto/crc32be-vx.S
+++ b/arch/s390/crypto/crc32be-vx.S
@@ -207,5 +207,6 @@
.Ldone:
VLGVF %r2,%v2,3
BR_EX %r14
+ENDPROC(crc32_be_vgfm_16)
.previous
diff --git a/arch/s390/crypto/crc32le-vx.S b/arch/s390/crypto/crc32le-vx.S
index 7d6f568..71caf0f 100644
--- a/arch/s390/crypto/crc32le-vx.S
+++ b/arch/s390/crypto/crc32le-vx.S
@@ -105,13 +105,14 @@
ENTRY(crc32_le_vgfm_16)
larl %r5,.Lconstants_CRC_32_LE
j crc32_le_vgfm_generic
+ENDPROC(crc32_le_vgfm_16)
ENTRY(crc32c_le_vgfm_16)
larl %r5,.Lconstants_CRC_32C_LE
j crc32_le_vgfm_generic
+ENDPROC(crc32c_le_vgfm_16)
-
-crc32_le_vgfm_generic:
+ENTRY(crc32_le_vgfm_generic)
/* Load CRC-32 constants */
VLM CONST_PERM_LE2BE,CONST_CRC_POLY,0,%r5
@@ -267,5 +268,6 @@
.Ldone:
VLGVF %r2,%v2,2
BR_EX %r14
+ENDPROC(crc32_le_vgfm_generic)
.previous
diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c
index 5346b5a..439b100 100644
--- a/arch/s390/crypto/des_s390.c
+++ b/arch/s390/crypto/des_s390.c
@@ -14,14 +14,15 @@
#include <linux/cpufeature.h>
#include <linux/crypto.h>
#include <linux/fips.h>
+#include <linux/mutex.h>
#include <crypto/algapi.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <asm/cpacf.h>
#define DES3_KEY_SIZE (3 * DES_KEY_SIZE)
static u8 *ctrblk;
-static DEFINE_SPINLOCK(ctrblk_lock);
+static DEFINE_MUTEX(ctrblk_lock);
static cpacf_mask_t km_functions, kmc_functions, kmctr_functions;
@@ -34,27 +35,24 @@
unsigned int key_len)
{
struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm);
- u32 tmp[DES_EXPKEY_WORDS];
+ int err;
- /* check for weak keys */
- if (!des_ekey(tmp, key) &&
- (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
- }
+ err = crypto_des_verify_key(tfm, key);
+ if (err)
+ return err;
memcpy(ctx->key, key, key_len);
return 0;
}
-static void des_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+static void s390_des_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm);
cpacf_km(CPACF_KM_DEA, ctx->key, out, in, DES_BLOCK_SIZE);
}
-static void des_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+static void s390_des_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm);
@@ -75,8 +73,8 @@
.cia_min_keysize = DES_KEY_SIZE,
.cia_max_keysize = DES_KEY_SIZE,
.cia_setkey = des_setkey,
- .cia_encrypt = des_encrypt,
- .cia_decrypt = des_decrypt,
+ .cia_encrypt = s390_des_encrypt,
+ .cia_decrypt = s390_des_decrypt,
}
}
};
@@ -224,24 +222,11 @@
unsigned int key_len)
{
struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm);
+ int err;
- if (!(crypto_memneq(key, &key[DES_KEY_SIZE], DES_KEY_SIZE) &&
- crypto_memneq(&key[DES_KEY_SIZE], &key[DES_KEY_SIZE * 2],
- DES_KEY_SIZE)) &&
- (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
- }
-
- /* in fips mode, ensure k1 != k2 and k2 != k3 and k1 != k3 */
- if (fips_enabled &&
- !(crypto_memneq(key, &key[DES_KEY_SIZE], DES_KEY_SIZE) &&
- crypto_memneq(&key[DES_KEY_SIZE], &key[DES_KEY_SIZE * 2],
- DES_KEY_SIZE) &&
- crypto_memneq(key, &key[DES_KEY_SIZE * 2], DES_KEY_SIZE))) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
- }
+ err = crypto_des3_ede_verify_key(tfm, key);
+ if (err)
+ return err;
memcpy(ctx->key, key, key_len);
return 0;
@@ -387,7 +372,7 @@
unsigned int n, nbytes;
int ret, locked;
- locked = spin_trylock(&ctrblk_lock);
+ locked = mutex_trylock(&ctrblk_lock);
ret = blkcipher_walk_virt_block(desc, walk, DES_BLOCK_SIZE);
while ((nbytes = walk->nbytes) >= DES_BLOCK_SIZE) {
@@ -404,7 +389,7 @@
ret = blkcipher_walk_done(desc, walk, nbytes - n);
}
if (locked)
- spin_unlock(&ctrblk_lock);
+ mutex_unlock(&ctrblk_lock);
/* final block may be < DES_BLOCK_SIZE, copy only nbytes */
if (nbytes) {
cpacf_kmctr(fc, ctx->key, buf, walk->src.virt.addr,
diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c
index 86aed30..a3e7400 100644
--- a/arch/s390/crypto/ghash_s390.c
+++ b/arch/s390/crypto/ghash_s390.c
@@ -137,7 +137,7 @@
static int __init ghash_mod_init(void)
{
if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH))
- return -EOPNOTSUPP;
+ return -ENODEV;
return crypto_register_shash(&ghash_alg);
}
@@ -153,4 +153,4 @@
MODULE_ALIAS_CRYPTO("ghash");
MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("GHASH Message Digest Algorithm, s390 implementation");
+MODULE_DESCRIPTION("GHASH hash function, s390 implementation");
diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c
index ab9a0eb..6184dce 100644
--- a/arch/s390/crypto/paes_s390.c
+++ b/arch/s390/crypto/paes_s390.c
@@ -5,7 +5,7 @@
* s390 implementation of the AES Cipher Algorithm with protected keys.
*
* s390 Version:
- * Copyright IBM Corp. 2017
+ * Copyright IBM Corp. 2017,2019
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
* Harald Freudenberger <freude@de.ibm.com>
*/
@@ -25,31 +25,79 @@
#include <asm/cpacf.h>
#include <asm/pkey.h>
+/*
+ * Key blobs smaller/bigger than these defines are rejected
+ * by the common code even before the individual setkey function
+ * is called. As paes can handle different kinds of key blobs
+ * and padding is also possible, the limits need to be generous.
+ */
+#define PAES_MIN_KEYSIZE 64
+#define PAES_MAX_KEYSIZE 256
+
static u8 *ctrblk;
static DEFINE_SPINLOCK(ctrblk_lock);
static cpacf_mask_t km_functions, kmc_functions, kmctr_functions;
+struct key_blob {
+ /*
+ * Small keys will be stored in the keybuf. Larger keys are
+ * stored in extra allocated memory. In both cases does
+ * key point to the memory where the key is stored.
+ * The code distinguishes by checking keylen against
+ * sizeof(keybuf). See the two following helper functions.
+ */
+ u8 *key;
+ u8 keybuf[128];
+ unsigned int keylen;
+};
+
+static inline int _copy_key_to_kb(struct key_blob *kb,
+ const u8 *key,
+ unsigned int keylen)
+{
+ if (keylen <= sizeof(kb->keybuf))
+ kb->key = kb->keybuf;
+ else {
+ kb->key = kmalloc(keylen, GFP_KERNEL);
+ if (!kb->key)
+ return -ENOMEM;
+ }
+ memcpy(kb->key, key, keylen);
+ kb->keylen = keylen;
+
+ return 0;
+}
+
+static inline void _free_kb_keybuf(struct key_blob *kb)
+{
+ if (kb->key && kb->key != kb->keybuf
+ && kb->keylen > sizeof(kb->keybuf)) {
+ kfree(kb->key);
+ kb->key = NULL;
+ }
+}
+
struct s390_paes_ctx {
- struct pkey_seckey sk;
+ struct key_blob kb;
struct pkey_protkey pk;
unsigned long fc;
};
struct s390_pxts_ctx {
- struct pkey_seckey sk[2];
+ struct key_blob kb[2];
struct pkey_protkey pk[2];
unsigned long fc;
};
-static inline int __paes_convert_key(struct pkey_seckey *sk,
+static inline int __paes_convert_key(struct key_blob *kb,
struct pkey_protkey *pk)
{
int i, ret;
/* try three times in case of failure */
for (i = 0; i < 3; i++) {
- ret = pkey_skey2pkey(sk, pk);
+ ret = pkey_keyblob2pkey(kb->key, kb->keylen, pk);
if (ret == 0)
break;
}
@@ -61,7 +109,7 @@
{
unsigned long fc;
- if (__paes_convert_key(&ctx->sk, &ctx->pk))
+ if (__paes_convert_key(&ctx->kb, &ctx->pk))
return -EINVAL;
/* Pick the correct function code based on the protected key type */
@@ -75,15 +123,33 @@
return ctx->fc ? 0 : -EINVAL;
}
-static int ecb_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
- unsigned int key_len)
+static int ecb_paes_init(struct crypto_tfm *tfm)
{
struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm);
- if (key_len != SECKEYBLOBSIZE)
- return -EINVAL;
+ ctx->kb.key = NULL;
- memcpy(ctx->sk.seckey, in_key, SECKEYBLOBSIZE);
+ return 0;
+}
+
+static void ecb_paes_exit(struct crypto_tfm *tfm)
+{
+ struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ _free_kb_keybuf(&ctx->kb);
+}
+
+static int ecb_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ int rc;
+ struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ _free_kb_keybuf(&ctx->kb);
+ rc = _copy_key_to_kb(&ctx->kb, in_key, key_len);
+ if (rc)
+ return rc;
+
if (__paes_set_key(ctx)) {
tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
@@ -145,10 +211,12 @@
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_list = LIST_HEAD_INIT(ecb_paes_alg.cra_list),
+ .cra_init = ecb_paes_init,
+ .cra_exit = ecb_paes_exit,
.cra_u = {
.blkcipher = {
- .min_keysize = SECKEYBLOBSIZE,
- .max_keysize = SECKEYBLOBSIZE,
+ .min_keysize = PAES_MIN_KEYSIZE,
+ .max_keysize = PAES_MAX_KEYSIZE,
.setkey = ecb_paes_set_key,
.encrypt = ecb_paes_encrypt,
.decrypt = ecb_paes_decrypt,
@@ -156,11 +224,27 @@
}
};
+static int cbc_paes_init(struct crypto_tfm *tfm)
+{
+ struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->kb.key = NULL;
+
+ return 0;
+}
+
+static void cbc_paes_exit(struct crypto_tfm *tfm)
+{
+ struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ _free_kb_keybuf(&ctx->kb);
+}
+
static int __cbc_paes_set_key(struct s390_paes_ctx *ctx)
{
unsigned long fc;
- if (__paes_convert_key(&ctx->sk, &ctx->pk))
+ if (__paes_convert_key(&ctx->kb, &ctx->pk))
return -EINVAL;
/* Pick the correct function code based on the protected key type */
@@ -177,9 +261,14 @@
static int cbc_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
unsigned int key_len)
{
+ int rc;
struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm);
- memcpy(ctx->sk.seckey, in_key, SECKEYBLOBSIZE);
+ _free_kb_keybuf(&ctx->kb);
+ rc = _copy_key_to_kb(&ctx->kb, in_key, key_len);
+ if (rc)
+ return rc;
+
if (__cbc_paes_set_key(ctx)) {
tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
@@ -248,10 +337,12 @@
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_list = LIST_HEAD_INIT(cbc_paes_alg.cra_list),
+ .cra_init = cbc_paes_init,
+ .cra_exit = cbc_paes_exit,
.cra_u = {
.blkcipher = {
- .min_keysize = SECKEYBLOBSIZE,
- .max_keysize = SECKEYBLOBSIZE,
+ .min_keysize = PAES_MIN_KEYSIZE,
+ .max_keysize = PAES_MAX_KEYSIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = cbc_paes_set_key,
.encrypt = cbc_paes_encrypt,
@@ -260,12 +351,30 @@
}
};
+static int xts_paes_init(struct crypto_tfm *tfm)
+{
+ struct s390_pxts_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->kb[0].key = NULL;
+ ctx->kb[1].key = NULL;
+
+ return 0;
+}
+
+static void xts_paes_exit(struct crypto_tfm *tfm)
+{
+ struct s390_pxts_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ _free_kb_keybuf(&ctx->kb[0]);
+ _free_kb_keybuf(&ctx->kb[1]);
+}
+
static int __xts_paes_set_key(struct s390_pxts_ctx *ctx)
{
unsigned long fc;
- if (__paes_convert_key(&ctx->sk[0], &ctx->pk[0]) ||
- __paes_convert_key(&ctx->sk[1], &ctx->pk[1]))
+ if (__paes_convert_key(&ctx->kb[0], &ctx->pk[0]) ||
+ __paes_convert_key(&ctx->kb[1], &ctx->pk[1]))
return -EINVAL;
if (ctx->pk[0].type != ctx->pk[1].type)
@@ -283,14 +392,27 @@
}
static int xts_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
- unsigned int key_len)
+ unsigned int xts_key_len)
{
+ int rc;
struct s390_pxts_ctx *ctx = crypto_tfm_ctx(tfm);
u8 ckey[2 * AES_MAX_KEY_SIZE];
- unsigned int ckey_len;
+ unsigned int ckey_len, key_len;
- memcpy(ctx->sk[0].seckey, in_key, SECKEYBLOBSIZE);
- memcpy(ctx->sk[1].seckey, in_key + SECKEYBLOBSIZE, SECKEYBLOBSIZE);
+ if (xts_key_len % 2)
+ return -EINVAL;
+
+ key_len = xts_key_len / 2;
+
+ _free_kb_keybuf(&ctx->kb[0]);
+ _free_kb_keybuf(&ctx->kb[1]);
+ rc = _copy_key_to_kb(&ctx->kb[0], in_key, key_len);
+ if (rc)
+ return rc;
+ rc = _copy_key_to_kb(&ctx->kb[1], in_key + key_len, key_len);
+ if (rc)
+ return rc;
+
if (__xts_paes_set_key(ctx)) {
tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
@@ -384,10 +506,12 @@
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_list = LIST_HEAD_INIT(xts_paes_alg.cra_list),
+ .cra_init = xts_paes_init,
+ .cra_exit = xts_paes_exit,
.cra_u = {
.blkcipher = {
- .min_keysize = 2 * SECKEYBLOBSIZE,
- .max_keysize = 2 * SECKEYBLOBSIZE,
+ .min_keysize = 2 * PAES_MIN_KEYSIZE,
+ .max_keysize = 2 * PAES_MAX_KEYSIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = xts_paes_set_key,
.encrypt = xts_paes_encrypt,
@@ -396,11 +520,27 @@
}
};
+static int ctr_paes_init(struct crypto_tfm *tfm)
+{
+ struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->kb.key = NULL;
+
+ return 0;
+}
+
+static void ctr_paes_exit(struct crypto_tfm *tfm)
+{
+ struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ _free_kb_keybuf(&ctx->kb);
+}
+
static int __ctr_paes_set_key(struct s390_paes_ctx *ctx)
{
unsigned long fc;
- if (__paes_convert_key(&ctx->sk, &ctx->pk))
+ if (__paes_convert_key(&ctx->kb, &ctx->pk))
return -EINVAL;
/* Pick the correct function code based on the protected key type */
@@ -418,9 +558,14 @@
static int ctr_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
unsigned int key_len)
{
+ int rc;
struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm);
- memcpy(ctx->sk.seckey, in_key, key_len);
+ _free_kb_keybuf(&ctx->kb);
+ rc = _copy_key_to_kb(&ctx->kb, in_key, key_len);
+ if (rc)
+ return rc;
+
if (__ctr_paes_set_key(ctx)) {
tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
@@ -530,10 +675,12 @@
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_list = LIST_HEAD_INIT(ctr_paes_alg.cra_list),
+ .cra_init = ctr_paes_init,
+ .cra_exit = ctr_paes_exit,
.cra_u = {
.blkcipher = {
- .min_keysize = SECKEYBLOBSIZE,
- .max_keysize = SECKEYBLOBSIZE,
+ .min_keysize = PAES_MIN_KEYSIZE,
+ .max_keysize = PAES_MAX_KEYSIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = ctr_paes_set_key,
.encrypt = ctr_paes_encrypt,
diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c
index a97a180..d977643 100644
--- a/arch/s390/crypto/prng.c
+++ b/arch/s390/crypto/prng.c
@@ -61,6 +61,7 @@
module_param_named(reseed_limit, prng_reseed_limit, int, 0);
MODULE_PARM_DESC(prng_reseed_limit, "PRNG reseed limit");
+static bool trng_available;
/*
* Any one who considers arithmetical methods of producing random digits is,
@@ -115,46 +116,68 @@
/*
* generate_entropy:
- * This algorithm produces 64 bytes of entropy data based on 1024
- * individual stckf() invocations assuming that each stckf() value
- * contributes 0.25 bits of entropy. So the caller gets 256 bit
- * entropy per 64 byte or 4 bits entropy per byte.
+ * This function fills a given buffer with random bytes. The entropy within
+ * the random bytes given back is assumed to have at least 50% - meaning
+ * a 64 bytes buffer has at least 64 * 8 / 2 = 256 bits of entropy.
+ * Within the function the entropy generation is done in junks of 64 bytes.
+ * So the caller should also ask for buffer fill in multiples of 64 bytes.
+ * The generation of the entropy is based on the assumption that every stckf()
+ * invocation produces 0.5 bits of entropy. To accumulate 256 bits of entropy
+ * at least 512 stckf() values are needed. The entropy relevant part of the
+ * stckf value is bit 51 (counting starts at the left with bit nr 0) so
+ * here we use the lower 4 bytes and exor the values into 2k of bufferspace.
+ * To be on the save side, if there is ever a problem with stckf() the
+ * other half of the page buffer is filled with bytes from urandom via
+ * get_random_bytes(), so this function consumes 2k of urandom for each
+ * requested 64 bytes output data. Finally the buffer page is condensed into
+ * a 64 byte value by hashing with a SHA512 hash.
*/
static int generate_entropy(u8 *ebuf, size_t nbytes)
{
int n, ret = 0;
- u8 *pg, *h, hash[64];
+ u8 *pg, pblock[80] = {
+ /* 8 x 64 bit init values */
+ 0x6A, 0x09, 0xE6, 0x67, 0xF3, 0xBC, 0xC9, 0x08,
+ 0xBB, 0x67, 0xAE, 0x85, 0x84, 0xCA, 0xA7, 0x3B,
+ 0x3C, 0x6E, 0xF3, 0x72, 0xFE, 0x94, 0xF8, 0x2B,
+ 0xA5, 0x4F, 0xF5, 0x3A, 0x5F, 0x1D, 0x36, 0xF1,
+ 0x51, 0x0E, 0x52, 0x7F, 0xAD, 0xE6, 0x82, 0xD1,
+ 0x9B, 0x05, 0x68, 0x8C, 0x2B, 0x3E, 0x6C, 0x1F,
+ 0x1F, 0x83, 0xD9, 0xAB, 0xFB, 0x41, 0xBD, 0x6B,
+ 0x5B, 0xE0, 0xCD, 0x19, 0x13, 0x7E, 0x21, 0x79,
+ /* 128 bit counter total message bit length */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00 };
- /* allocate 2 pages */
- pg = (u8 *) __get_free_pages(GFP_KERNEL, 1);
+ /* allocate one page stckf buffer */
+ pg = (u8 *) __get_free_page(GFP_KERNEL);
if (!pg) {
prng_errorflag = PRNG_GEN_ENTROPY_FAILED;
return -ENOMEM;
}
+ /* fill the ebuf in chunks of 64 byte each */
while (nbytes) {
- /* fill pages with urandom bytes */
- get_random_bytes(pg, 2*PAGE_SIZE);
- /* exor pages with 1024 stckf values */
- for (n = 0; n < 2 * PAGE_SIZE / sizeof(u64); n++) {
- u64 *p = ((u64 *)pg) + n;
+ /* fill lower 2k with urandom bytes */
+ get_random_bytes(pg, PAGE_SIZE / 2);
+ /* exor upper 2k with 512 stckf values, offset 4 bytes each */
+ for (n = 0; n < 512; n++) {
+ int offset = (PAGE_SIZE / 2) + (n * 4) - 4;
+ u64 *p = (u64 *)(pg + offset);
*p ^= get_tod_clock_fast();
}
- n = (nbytes < sizeof(hash)) ? nbytes : sizeof(hash);
- if (n < sizeof(hash))
- h = hash;
- else
- h = ebuf;
- /* hash over the filled pages */
- cpacf_kimd(CPACF_KIMD_SHA_512, h, pg, 2*PAGE_SIZE);
- if (n < sizeof(hash))
- memcpy(ebuf, hash, n);
+ /* hash over the filled page */
+ cpacf_klmd(CPACF_KLMD_SHA_512, pblock, pg, PAGE_SIZE);
+ n = (nbytes < 64) ? nbytes : 64;
+ memcpy(ebuf, pblock, n);
ret += n;
ebuf += n;
nbytes -= n;
}
- free_pages((unsigned long)pg, 1);
+ memzero_explicit(pblock, sizeof(pblock));
+ memzero_explicit(pg, PAGE_SIZE);
+ free_page((unsigned long)pg);
return ret;
}
@@ -344,8 +367,8 @@
static int __init prng_sha512_instantiate(void)
{
- int ret, datalen;
- u8 seed[64 + 32 + 16];
+ int ret, datalen, seedlen;
+ u8 seed[128 + 16];
pr_debug("prng runs in SHA-512 mode "
"with chunksize=%d and reseed_limit=%u\n",
@@ -368,16 +391,36 @@
if (ret)
goto outfree;
- /* generate initial seed bytestring, with 256 + 128 bits entropy */
- ret = generate_entropy(seed, 64 + 32);
- if (ret != 64 + 32)
- goto outfree;
- /* followed by 16 bytes of unique nonce */
- get_tod_clock_ext(seed + 64 + 32);
+ /* generate initial seed, we need at least 256 + 128 bits entropy. */
+ if (trng_available) {
+ /*
+ * Trng available, so use it. The trng works in chunks of
+ * 32 bytes and produces 100% entropy. So we pull 64 bytes
+ * which gives us 512 bits entropy.
+ */
+ seedlen = 2 * 32;
+ cpacf_trng(NULL, 0, seed, seedlen);
+ } else {
+ /*
+ * No trng available, so use the generate_entropy() function.
+ * This function works in 64 byte junks and produces
+ * 50% entropy. So we pull 2*64 bytes which gives us 512 bits
+ * of entropy.
+ */
+ seedlen = 2 * 64;
+ ret = generate_entropy(seed, seedlen);
+ if (ret != seedlen)
+ goto outfree;
+ }
- /* initial seed of the prno drng */
+ /* append the seed by 16 bytes of unique nonce */
+ get_tod_clock_ext(seed + seedlen);
+ seedlen += 16;
+
+ /* now initial seed of the prno drng */
cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED,
- &prng_data->prnows, NULL, 0, seed, sizeof(seed));
+ &prng_data->prnows, NULL, 0, seed, seedlen);
+ memzero_explicit(seed, sizeof(seed));
/* if fips mode is enabled, generate a first block of random
bytes for the FIPS 140-2 Conditional Self Test */
@@ -405,17 +448,26 @@
static int prng_sha512_reseed(void)
{
- int ret;
+ int ret, seedlen;
u8 seed[64];
- /* fetch 256 bits of fresh entropy */
- ret = generate_entropy(seed, sizeof(seed));
- if (ret != sizeof(seed))
- return ret;
+ /* We need at least 256 bits of fresh entropy for reseeding */
+ if (trng_available) {
+ /* trng produces 256 bits entropy in 32 bytes */
+ seedlen = 32;
+ cpacf_trng(NULL, 0, seed, seedlen);
+ } else {
+ /* generate_entropy() produces 256 bits entropy in 64 bytes */
+ seedlen = 64;
+ ret = generate_entropy(seed, seedlen);
+ if (ret != sizeof(seed))
+ return ret;
+ }
/* do a reseed of the prno drng with this bytestring */
cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED,
- &prng_data->prnows, NULL, 0, seed, sizeof(seed));
+ &prng_data->prnows, NULL, 0, seed, seedlen);
+ memzero_explicit(seed, sizeof(seed));
return 0;
}
@@ -592,6 +644,7 @@
ret = -EFAULT;
break;
}
+ memzero_explicit(p, n);
ubuf += n;
nbytes -= n;
ret += n;
@@ -771,7 +824,11 @@
/* check if the CPU has a PRNG */
if (!cpacf_query_func(CPACF_KMC, CPACF_KMC_PRNG))
- return -EOPNOTSUPP;
+ return -ENODEV;
+
+ /* check if TRNG subfunction is available */
+ if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG))
+ trng_available = true;
/* choose prng mode */
if (prng_mode != PRNG_MODE_TDES) {
@@ -780,7 +837,7 @@
if (prng_mode == PRNG_MODE_SHA512) {
pr_err("The prng module cannot "
"start in SHA-512 mode\n");
- return -EOPNOTSUPP;
+ return -ENODEV;
}
prng_mode = PRNG_MODE_TDES;
} else
diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h
index d6f8258..ada2f98 100644
--- a/arch/s390/crypto/sha.h
+++ b/arch/s390/crypto/sha.h
@@ -12,15 +12,17 @@
#include <linux/crypto.h>
#include <crypto/sha.h>
+#include <crypto/sha3.h>
/* must be big enough for the largest SHA variant */
-#define SHA_MAX_STATE_SIZE (SHA512_DIGEST_SIZE / 4)
-#define SHA_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
+#define SHA3_STATE_SIZE 200
+#define CPACF_MAX_PARMBLOCK_SIZE SHA3_STATE_SIZE
+#define SHA_MAX_BLOCK_SIZE SHA3_224_BLOCK_SIZE
struct s390_sha_ctx {
- u64 count; /* message length in bytes */
- u32 state[SHA_MAX_STATE_SIZE];
- u8 buf[2 * SHA_MAX_BLOCK_SIZE];
+ u64 count; /* message length in bytes */
+ u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)];
+ u8 buf[SHA_MAX_BLOCK_SIZE];
int func; /* KIMD function to use */
};
diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c
index 009572e..7c15542 100644
--- a/arch/s390/crypto/sha1_s390.c
+++ b/arch/s390/crypto/sha1_s390.c
@@ -86,7 +86,7 @@
static int __init sha1_s390_init(void)
{
if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_1))
- return -EOPNOTSUPP;
+ return -ENODEV;
return crypto_register_shash(&alg);
}
diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c
index 62833a1..b52c87e 100644
--- a/arch/s390/crypto/sha256_s390.c
+++ b/arch/s390/crypto/sha256_s390.c
@@ -17,7 +17,7 @@
#include "sha.h"
-static int sha256_init(struct shash_desc *desc)
+static int s390_sha256_init(struct shash_desc *desc)
{
struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
@@ -60,7 +60,7 @@
static struct shash_alg sha256_alg = {
.digestsize = SHA256_DIGEST_SIZE,
- .init = sha256_init,
+ .init = s390_sha256_init,
.update = s390_sha_update,
.final = s390_sha_final,
.export = sha256_export,
@@ -76,7 +76,7 @@
}
};
-static int sha224_init(struct shash_desc *desc)
+static int s390_sha224_init(struct shash_desc *desc)
{
struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
@@ -96,7 +96,7 @@
static struct shash_alg sha224_alg = {
.digestsize = SHA224_DIGEST_SIZE,
- .init = sha224_init,
+ .init = s390_sha224_init,
.update = s390_sha_update,
.final = s390_sha_final,
.export = sha256_export,
@@ -117,7 +117,7 @@
int ret;
if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_256))
- return -EOPNOTSUPP;
+ return -ENODEV;
ret = crypto_register_shash(&sha256_alg);
if (ret < 0)
goto out;
diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c
new file mode 100644
index 0000000..460cbbb
--- /dev/null
+++ b/arch/s390/crypto/sha3_256_s390.c
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Cryptographic API.
+ *
+ * s390 implementation of the SHA256 and SHA224 Secure Hash Algorithm.
+ *
+ * s390 Version:
+ * Copyright IBM Corp. 2019
+ * Author(s): Joerg Schmidbauer (jschmidb@de.ibm.com)
+ */
+#include <crypto/internal/hash.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/cpufeature.h>
+#include <crypto/sha.h>
+#include <crypto/sha3.h>
+#include <asm/cpacf.h>
+
+#include "sha.h"
+
+static int sha3_256_init(struct shash_desc *desc)
+{
+ struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
+
+ memset(sctx->state, 0, sizeof(sctx->state));
+ sctx->count = 0;
+ sctx->func = CPACF_KIMD_SHA3_256;
+
+ return 0;
+}
+
+static int sha3_256_export(struct shash_desc *desc, void *out)
+{
+ struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
+ struct sha3_state *octx = out;
+
+ octx->rsiz = sctx->count;
+ memcpy(octx->st, sctx->state, sizeof(octx->st));
+ memcpy(octx->buf, sctx->buf, sizeof(octx->buf));
+
+ return 0;
+}
+
+static int sha3_256_import(struct shash_desc *desc, const void *in)
+{
+ struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
+ const struct sha3_state *ictx = in;
+
+ sctx->count = ictx->rsiz;
+ memcpy(sctx->state, ictx->st, sizeof(ictx->st));
+ memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
+ sctx->func = CPACF_KIMD_SHA3_256;
+
+ return 0;
+}
+
+static int sha3_224_import(struct shash_desc *desc, const void *in)
+{
+ struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
+ const struct sha3_state *ictx = in;
+
+ sctx->count = ictx->rsiz;
+ memcpy(sctx->state, ictx->st, sizeof(ictx->st));
+ memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
+ sctx->func = CPACF_KIMD_SHA3_224;
+
+ return 0;
+}
+
+static struct shash_alg sha3_256_alg = {
+ .digestsize = SHA3_256_DIGEST_SIZE, /* = 32 */
+ .init = sha3_256_init,
+ .update = s390_sha_update,
+ .final = s390_sha_final,
+ .export = sha3_256_export,
+ .import = sha3_256_import,
+ .descsize = sizeof(struct s390_sha_ctx),
+ .statesize = sizeof(struct sha3_state),
+ .base = {
+ .cra_name = "sha3-256",
+ .cra_driver_name = "sha3-256-s390",
+ .cra_priority = 300,
+ .cra_blocksize = SHA3_256_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ }
+};
+
+static int sha3_224_init(struct shash_desc *desc)
+{
+ struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
+
+ memset(sctx->state, 0, sizeof(sctx->state));
+ sctx->count = 0;
+ sctx->func = CPACF_KIMD_SHA3_224;
+
+ return 0;
+}
+
+static struct shash_alg sha3_224_alg = {
+ .digestsize = SHA3_224_DIGEST_SIZE,
+ .init = sha3_224_init,
+ .update = s390_sha_update,
+ .final = s390_sha_final,
+ .export = sha3_256_export, /* same as for 256 */
+ .import = sha3_224_import, /* function code different! */
+ .descsize = sizeof(struct s390_sha_ctx),
+ .statesize = sizeof(struct sha3_state),
+ .base = {
+ .cra_name = "sha3-224",
+ .cra_driver_name = "sha3-224-s390",
+ .cra_priority = 300,
+ .cra_blocksize = SHA3_224_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ }
+};
+
+static int __init sha3_256_s390_init(void)
+{
+ int ret;
+
+ if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA3_256))
+ return -ENODEV;
+
+ ret = crypto_register_shash(&sha3_256_alg);
+ if (ret < 0)
+ goto out;
+
+ ret = crypto_register_shash(&sha3_224_alg);
+ if (ret < 0)
+ crypto_unregister_shash(&sha3_256_alg);
+out:
+ return ret;
+}
+
+static void __exit sha3_256_s390_fini(void)
+{
+ crypto_unregister_shash(&sha3_224_alg);
+ crypto_unregister_shash(&sha3_256_alg);
+}
+
+module_cpu_feature_match(MSA, sha3_256_s390_init);
+module_exit(sha3_256_s390_fini);
+
+MODULE_ALIAS_CRYPTO("sha3-256");
+MODULE_ALIAS_CRYPTO("sha3-224");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("SHA3-256 and SHA3-224 Secure Hash Algorithm");
diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c
new file mode 100644
index 0000000..72cf460
--- /dev/null
+++ b/arch/s390/crypto/sha3_512_s390.c
@@ -0,0 +1,155 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Cryptographic API.
+ *
+ * s390 implementation of the SHA512 and SHA384 Secure Hash Algorithm.
+ *
+ * Copyright IBM Corp. 2019
+ * Author(s): Joerg Schmidbauer (jschmidb@de.ibm.com)
+ */
+#include <crypto/internal/hash.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/cpufeature.h>
+#include <crypto/sha.h>
+#include <crypto/sha3.h>
+#include <asm/cpacf.h>
+
+#include "sha.h"
+
+static int sha3_512_init(struct shash_desc *desc)
+{
+ struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
+
+ memset(sctx->state, 0, sizeof(sctx->state));
+ sctx->count = 0;
+ sctx->func = CPACF_KIMD_SHA3_512;
+
+ return 0;
+}
+
+static int sha3_512_export(struct shash_desc *desc, void *out)
+{
+ struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
+ struct sha3_state *octx = out;
+
+ octx->rsiz = sctx->count;
+ octx->rsizw = sctx->count >> 32;
+
+ memcpy(octx->st, sctx->state, sizeof(octx->st));
+ memcpy(octx->buf, sctx->buf, sizeof(octx->buf));
+
+ return 0;
+}
+
+static int sha3_512_import(struct shash_desc *desc, const void *in)
+{
+ struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
+ const struct sha3_state *ictx = in;
+
+ if (unlikely(ictx->rsizw))
+ return -ERANGE;
+ sctx->count = ictx->rsiz;
+
+ memcpy(sctx->state, ictx->st, sizeof(ictx->st));
+ memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
+ sctx->func = CPACF_KIMD_SHA3_512;
+
+ return 0;
+}
+
+static int sha3_384_import(struct shash_desc *desc, const void *in)
+{
+ struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
+ const struct sha3_state *ictx = in;
+
+ if (unlikely(ictx->rsizw))
+ return -ERANGE;
+ sctx->count = ictx->rsiz;
+
+ memcpy(sctx->state, ictx->st, sizeof(ictx->st));
+ memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
+ sctx->func = CPACF_KIMD_SHA3_384;
+
+ return 0;
+}
+
+static struct shash_alg sha3_512_alg = {
+ .digestsize = SHA3_512_DIGEST_SIZE,
+ .init = sha3_512_init,
+ .update = s390_sha_update,
+ .final = s390_sha_final,
+ .export = sha3_512_export,
+ .import = sha3_512_import,
+ .descsize = sizeof(struct s390_sha_ctx),
+ .statesize = sizeof(struct sha3_state),
+ .base = {
+ .cra_name = "sha3-512",
+ .cra_driver_name = "sha3-512-s390",
+ .cra_priority = 300,
+ .cra_blocksize = SHA3_512_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ }
+};
+
+MODULE_ALIAS_CRYPTO("sha3-512");
+
+static int sha3_384_init(struct shash_desc *desc)
+{
+ struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
+
+ memset(sctx->state, 0, sizeof(sctx->state));
+ sctx->count = 0;
+ sctx->func = CPACF_KIMD_SHA3_384;
+
+ return 0;
+}
+
+static struct shash_alg sha3_384_alg = {
+ .digestsize = SHA3_384_DIGEST_SIZE,
+ .init = sha3_384_init,
+ .update = s390_sha_update,
+ .final = s390_sha_final,
+ .export = sha3_512_export, /* same as for 512 */
+ .import = sha3_384_import, /* function code different! */
+ .descsize = sizeof(struct s390_sha_ctx),
+ .statesize = sizeof(struct sha3_state),
+ .base = {
+ .cra_name = "sha3-384",
+ .cra_driver_name = "sha3-384-s390",
+ .cra_priority = 300,
+ .cra_blocksize = SHA3_384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct s390_sha_ctx),
+ .cra_module = THIS_MODULE,
+ }
+};
+
+MODULE_ALIAS_CRYPTO("sha3-384");
+
+static int __init init(void)
+{
+ int ret;
+
+ if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA3_512))
+ return -ENODEV;
+ ret = crypto_register_shash(&sha3_512_alg);
+ if (ret < 0)
+ goto out;
+ ret = crypto_register_shash(&sha3_384_alg);
+ if (ret < 0)
+ crypto_unregister_shash(&sha3_512_alg);
+out:
+ return ret;
+}
+
+static void __exit fini(void)
+{
+ crypto_unregister_shash(&sha3_512_alg);
+ crypto_unregister_shash(&sha3_384_alg);
+}
+
+module_cpu_feature_match(MSA, init);
+module_exit(fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("SHA3-512 and SHA3-384 Secure Hash Algorithm");
diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c
index be589c3..ad29db0 100644
--- a/arch/s390/crypto/sha512_s390.c
+++ b/arch/s390/crypto/sha512_s390.c
@@ -127,7 +127,7 @@
int ret;
if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_512))
- return -EOPNOTSUPP;
+ return -ENODEV;
if ((ret = crypto_register_shash(&sha512_alg)) < 0)
goto out;
if ((ret = crypto_register_shash(&sha384_alg)) < 0)
diff --git a/arch/s390/crypto/sha_common.c b/arch/s390/crypto/sha_common.c
index cf0718d..d39e0f0 100644
--- a/arch/s390/crypto/sha_common.c
+++ b/arch/s390/crypto/sha_common.c
@@ -20,7 +20,7 @@
unsigned int index, n;
/* how much is already in the buffer? */
- index = ctx->count & (bsize - 1);
+ index = ctx->count % bsize;
ctx->count += len;
if ((index + len) < bsize)
@@ -37,7 +37,7 @@
/* process as many blocks as possible */
if (len >= bsize) {
- n = len & ~(bsize - 1);
+ n = (len / bsize) * bsize;
cpacf_kimd(ctx->func, ctx->state, data, n);
data += n;
len -= n;
@@ -50,34 +50,63 @@
}
EXPORT_SYMBOL_GPL(s390_sha_update);
+static int s390_crypto_shash_parmsize(int func)
+{
+ switch (func) {
+ case CPACF_KLMD_SHA_1:
+ return 20;
+ case CPACF_KLMD_SHA_256:
+ return 32;
+ case CPACF_KLMD_SHA_512:
+ return 64;
+ case CPACF_KLMD_SHA3_224:
+ case CPACF_KLMD_SHA3_256:
+ case CPACF_KLMD_SHA3_384:
+ case CPACF_KLMD_SHA3_512:
+ return 200;
+ default:
+ return -EINVAL;
+ }
+}
+
int s390_sha_final(struct shash_desc *desc, u8 *out)
{
struct s390_sha_ctx *ctx = shash_desc_ctx(desc);
unsigned int bsize = crypto_shash_blocksize(desc->tfm);
u64 bits;
- unsigned int index, end, plen;
+ unsigned int n, mbl_offset;
- /* SHA-512 uses 128 bit padding length */
- plen = (bsize > SHA256_BLOCK_SIZE) ? 16 : 8;
-
- /* must perform manual padding */
- index = ctx->count & (bsize - 1);
- end = (index < bsize - plen) ? bsize : (2 * bsize);
-
- /* start pad with 1 */
- ctx->buf[index] = 0x80;
- index++;
-
- /* pad with zeros */
- memset(ctx->buf + index, 0x00, end - index - 8);
-
- /*
- * Append message length. Well, SHA-512 wants a 128 bit length value,
- * nevertheless we use u64, should be enough for now...
- */
+ n = ctx->count % bsize;
bits = ctx->count * 8;
- memcpy(ctx->buf + end - 8, &bits, sizeof(bits));
- cpacf_kimd(ctx->func, ctx->state, ctx->buf, end);
+ mbl_offset = s390_crypto_shash_parmsize(ctx->func) / sizeof(u32);
+ if (mbl_offset < 0)
+ return -EINVAL;
+
+ /* set total msg bit length (mbl) in CPACF parmblock */
+ switch (ctx->func) {
+ case CPACF_KLMD_SHA_1:
+ case CPACF_KLMD_SHA_256:
+ memcpy(ctx->state + mbl_offset, &bits, sizeof(bits));
+ break;
+ case CPACF_KLMD_SHA_512:
+ /*
+ * the SHA512 parmblock has a 128-bit mbl field, clear
+ * high-order u64 field, copy bits to low-order u64 field
+ */
+ memset(ctx->state + mbl_offset, 0x00, sizeof(bits));
+ mbl_offset += sizeof(u64) / sizeof(u32);
+ memcpy(ctx->state + mbl_offset, &bits, sizeof(bits));
+ break;
+ case CPACF_KLMD_SHA3_224:
+ case CPACF_KLMD_SHA3_256:
+ case CPACF_KLMD_SHA3_384:
+ case CPACF_KLMD_SHA3_512:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ cpacf_klmd(ctx->func, ctx->state, ctx->buf, n);
/* copy digest to out */
memcpy(out, ctx->state, crypto_shash_digestsize(desc->tfm));