diff options
Diffstat (limited to 'arch/s390/crypto')
-rw-r--r-- | arch/s390/crypto/aes_s390.c | 590 | ||||
-rw-r--r-- | arch/s390/crypto/crc32-vx.c | 2 | ||||
-rw-r--r-- | arch/s390/crypto/des_s390.c | 330 | ||||
-rw-r--r-- | arch/s390/crypto/ghash_s390.c | 20 | ||||
-rw-r--r-- | arch/s390/crypto/prng.c | 101 | ||||
-rw-r--r-- | arch/s390/crypto/sha1_s390.c | 2 | ||||
-rw-r--r-- | arch/s390/crypto/sha256_s390.c | 2 | ||||
-rw-r--r-- | arch/s390/crypto/sha512_s390.c | 2 | ||||
-rw-r--r-- | arch/s390/crypto/sha_common.c | 23 |
9 files changed, 409 insertions, 663 deletions
diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 2ea18b050309..303d28eb03a2 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -31,69 +31,29 @@ #include <crypto/xts.h> #include <asm/cpacf.h> -#define AES_KEYLEN_128 1 -#define AES_KEYLEN_192 2 -#define AES_KEYLEN_256 4 - static u8 *ctrblk; static DEFINE_SPINLOCK(ctrblk_lock); -static char keylen_flag; + +static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; struct s390_aes_ctx { u8 key[AES_MAX_KEY_SIZE]; - long enc; - long dec; int key_len; + unsigned long fc; union { struct crypto_skcipher *blk; struct crypto_cipher *cip; } fallback; }; -struct pcc_param { - u8 key[32]; - u8 tweak[16]; - u8 block[16]; - u8 bit[16]; - u8 xts[16]; -}; - struct s390_xts_ctx { u8 key[32]; u8 pcc_key[32]; - long enc; - long dec; int key_len; + unsigned long fc; struct crypto_skcipher *fallback; }; -/* - * Check if the key_len is supported by the HW. - * Returns 0 if it is, a positive number if it is not and software fallback is - * required or a negative number in case the key size is not valid - */ -static int need_fallback(unsigned int key_len) -{ - switch (key_len) { - case 16: - if (!(keylen_flag & AES_KEYLEN_128)) - return 1; - break; - case 24: - if (!(keylen_flag & AES_KEYLEN_192)) - return 1; - break; - case 32: - if (!(keylen_flag & AES_KEYLEN_256)) - return 1; - break; - default: - return -1; - break; - } - return 0; -} - static int setkey_fallback_cip(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) { @@ -117,72 +77,44 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) { struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - u32 *flags = &tfm->crt_flags; - int ret; + unsigned long fc; - ret = need_fallback(key_len); - if (ret < 0) { - *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; - return -EINVAL; - } + /* Pick the correct function code based on the key length */ + fc = (key_len == 16) ? CPACF_KM_AES_128 : + (key_len == 24) ? CPACF_KM_AES_192 : + (key_len == 32) ? CPACF_KM_AES_256 : 0; - sctx->key_len = key_len; - if (!ret) { - memcpy(sctx->key, in_key, key_len); - return 0; - } + /* Check if the function code is available */ + sctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; + if (!sctx->fc) + return setkey_fallback_cip(tfm, in_key, key_len); - return setkey_fallback_cip(tfm, in_key, key_len); + sctx->key_len = key_len; + memcpy(sctx->key, in_key, key_len); + return 0; } static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - if (unlikely(need_fallback(sctx->key_len))) { + if (unlikely(!sctx->fc)) { crypto_cipher_encrypt_one(sctx->fallback.cip, out, in); return; } - - switch (sctx->key_len) { - case 16: - cpacf_km(CPACF_KM_AES_128_ENC, &sctx->key, out, in, - AES_BLOCK_SIZE); - break; - case 24: - cpacf_km(CPACF_KM_AES_192_ENC, &sctx->key, out, in, - AES_BLOCK_SIZE); - break; - case 32: - cpacf_km(CPACF_KM_AES_256_ENC, &sctx->key, out, in, - AES_BLOCK_SIZE); - break; - } + cpacf_km(sctx->fc, &sctx->key, out, in, AES_BLOCK_SIZE); } static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - if (unlikely(need_fallback(sctx->key_len))) { + if (unlikely(!sctx->fc)) { crypto_cipher_decrypt_one(sctx->fallback.cip, out, in); return; } - - switch (sctx->key_len) { - case 16: - cpacf_km(CPACF_KM_AES_128_DEC, &sctx->key, out, in, - AES_BLOCK_SIZE); - break; - case 24: - cpacf_km(CPACF_KM_AES_192_DEC, &sctx->key, out, in, - AES_BLOCK_SIZE); - break; - case 32: - cpacf_km(CPACF_KM_AES_256_DEC, &sctx->key, out, in, - AES_BLOCK_SIZE); - break; - } + cpacf_km(sctx->fc | CPACF_DECRYPT, + &sctx->key, out, in, AES_BLOCK_SIZE); } static int fallback_init_cip(struct crypto_tfm *tfm) @@ -291,50 +223,37 @@ static int ecb_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) { struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - int ret; + unsigned long fc; - ret = need_fallback(key_len); - if (ret > 0) { - sctx->key_len = key_len; - return setkey_fallback_blk(tfm, in_key, key_len); - } + /* Pick the correct function code based on the key length */ + fc = (key_len == 16) ? CPACF_KM_AES_128 : + (key_len == 24) ? CPACF_KM_AES_192 : + (key_len == 32) ? CPACF_KM_AES_256 : 0; - switch (key_len) { - case 16: - sctx->enc = CPACF_KM_AES_128_ENC; - sctx->dec = CPACF_KM_AES_128_DEC; - break; - case 24: - sctx->enc = CPACF_KM_AES_192_ENC; - sctx->dec = CPACF_KM_AES_192_DEC; - break; - case 32: - sctx->enc = CPACF_KM_AES_256_ENC; - sctx->dec = CPACF_KM_AES_256_DEC; - break; - } + /* Check if the function code is available */ + sctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; + if (!sctx->fc) + return setkey_fallback_blk(tfm, in_key, key_len); - return aes_set_key(tfm, in_key, key_len); + sctx->key_len = key_len; + memcpy(sctx->key, in_key, key_len); + return 0; } -static int ecb_aes_crypt(struct blkcipher_desc *desc, long func, void *param, +static int ecb_aes_crypt(struct blkcipher_desc *desc, unsigned long modifier, struct blkcipher_walk *walk) { - int ret = blkcipher_walk_virt(desc, walk); - unsigned int nbytes; + struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int nbytes, n; + int ret; - while ((nbytes = walk->nbytes)) { + ret = blkcipher_walk_virt(desc, walk); + while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { /* only use complete blocks */ - unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1); - u8 *out = walk->dst.virt.addr; - u8 *in = walk->src.virt.addr; - - ret = cpacf_km(func, param, out, in, n); - if (ret < 0 || ret != n) - return -EIO; - - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, walk, nbytes); + n = nbytes & ~(AES_BLOCK_SIZE - 1); + cpacf_km(sctx->fc | modifier, sctx->key, + walk->dst.virt.addr, walk->src.virt.addr, n); + ret = blkcipher_walk_done(desc, walk, nbytes - n); } return ret; @@ -347,11 +266,11 @@ static int ecb_aes_encrypt(struct blkcipher_desc *desc, struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; - if (unlikely(need_fallback(sctx->key_len))) + if (unlikely(!sctx->fc)) return fallback_blk_enc(desc, dst, src, nbytes); blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_aes_crypt(desc, sctx->enc, sctx->key, &walk); + return ecb_aes_crypt(desc, 0, &walk); } static int ecb_aes_decrypt(struct blkcipher_desc *desc, @@ -361,11 +280,11 @@ static int ecb_aes_decrypt(struct blkcipher_desc *desc, struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; - if (unlikely(need_fallback(sctx->key_len))) + if (unlikely(!sctx->fc)) return fallback_blk_dec(desc, dst, src, nbytes); blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_aes_crypt(desc, sctx->dec, sctx->key, &walk); + return ecb_aes_crypt(desc, CPACF_DECRYPT, &walk); } static int fallback_init_blk(struct crypto_tfm *tfm) @@ -420,64 +339,45 @@ static int cbc_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) { struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - int ret; + unsigned long fc; - ret = need_fallback(key_len); - if (ret > 0) { - sctx->key_len = key_len; - return setkey_fallback_blk(tfm, in_key, key_len); - } + /* Pick the correct function code based on the key length */ + fc = (key_len == 16) ? CPACF_KMC_AES_128 : + (key_len == 24) ? CPACF_KMC_AES_192 : + (key_len == 32) ? CPACF_KMC_AES_256 : 0; - switch (key_len) { - case 16: - sctx->enc = CPACF_KMC_AES_128_ENC; - sctx->dec = CPACF_KMC_AES_128_DEC; - break; - case 24: - sctx->enc = CPACF_KMC_AES_192_ENC; - sctx->dec = CPACF_KMC_AES_192_DEC; - break; - case 32: - sctx->enc = CPACF_KMC_AES_256_ENC; - sctx->dec = CPACF_KMC_AES_256_DEC; - break; - } + /* Check if the function code is available */ + sctx->fc = (fc && cpacf_test_func(&kmc_functions, fc)) ? fc : 0; + if (!sctx->fc) + return setkey_fallback_blk(tfm, in_key, key_len); - return aes_set_key(tfm, in_key, key_len); + sctx->key_len = key_len; + memcpy(sctx->key, in_key, key_len); + return 0; } -static int cbc_aes_crypt(struct blkcipher_desc *desc, long func, +static int cbc_aes_crypt(struct blkcipher_desc *desc, unsigned long modifier, struct blkcipher_walk *walk) { struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); - int ret = blkcipher_walk_virt(desc, walk); - unsigned int nbytes = walk->nbytes; + unsigned int nbytes, n; + int ret; struct { u8 iv[AES_BLOCK_SIZE]; u8 key[AES_MAX_KEY_SIZE]; } param; - if (!nbytes) - goto out; - + ret = blkcipher_walk_virt(desc, walk); memcpy(param.iv, walk->iv, AES_BLOCK_SIZE); memcpy(param.key, sctx->key, sctx->key_len); - do { + while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { /* only use complete blocks */ - unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1); - u8 *out = walk->dst.virt.addr; - u8 *in = walk->src.virt.addr; - - ret = cpacf_kmc(func, ¶m, out, in, n); - if (ret < 0 || ret != n) - return -EIO; - - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, walk, nbytes); - } while ((nbytes = walk->nbytes)); + n = nbytes & ~(AES_BLOCK_SIZE - 1); + cpacf_kmc(sctx->fc | modifier, ¶m, + walk->dst.virt.addr, walk->src.virt.addr, n); + ret = blkcipher_walk_done(desc, walk, nbytes - n); + } memcpy(walk->iv, param.iv, AES_BLOCK_SIZE); - -out: return ret; } @@ -488,11 +388,11 @@ static int cbc_aes_encrypt(struct blkcipher_desc *desc, struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; - if (unlikely(need_fallback(sctx->key_len))) + if (unlikely(!sctx->fc)) return fallback_blk_enc(desc, dst, src, nbytes); blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_aes_crypt(desc, sctx->enc, &walk); + return cbc_aes_crypt(desc, 0, &walk); } static int cbc_aes_decrypt(struct blkcipher_desc *desc, @@ -502,11 +402,11 @@ static int cbc_aes_decrypt(struct blkcipher_desc *desc, struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; - if (unlikely(need_fallback(sctx->key_len))) + if (unlikely(!sctx->fc)) return fallback_blk_dec(desc, dst, src, nbytes); blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_aes_crypt(desc, sctx->dec, &walk); + return cbc_aes_crypt(desc, CPACF_DECRYPT, &walk); } static struct crypto_alg cbc_aes_alg = { @@ -594,83 +494,67 @@ static int xts_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) { struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm); - u32 *flags = &tfm->crt_flags; + unsigned long fc; int err; err = xts_check_key(tfm, in_key, key_len); if (err) return err; - switch (key_len) { - case 32: - xts_ctx->enc = CPACF_KM_XTS_128_ENC; - xts_ctx->dec = CPACF_KM_XTS_128_DEC; - memcpy(xts_ctx->key + 16, in_key, 16); - memcpy(xts_ctx->pcc_key + 16, in_key + 16, 16); - break; - case 48: - xts_ctx->enc = 0; - xts_ctx->dec = 0; - xts_fallback_setkey(tfm, in_key, key_len); - break; - case 64: - xts_ctx->enc = CPACF_KM_XTS_256_ENC; - xts_ctx->dec = CPACF_KM_XTS_256_DEC; - memcpy(xts_ctx->key, in_key, 32); - memcpy(xts_ctx->pcc_key, in_key + 32, 32); - break; - default: - *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; - return -EINVAL; - } + /* Pick the correct function code based on the key length */ + fc = (key_len == 32) ? CPACF_KM_XTS_128 : + (key_len == 64) ? CPACF_KM_XTS_256 : 0; + + /* Check if the function code is available */ + xts_ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; + if (!xts_ctx->fc) + return xts_fallback_setkey(tfm, in_key, key_len); + + /* Split the XTS key into the two subkeys */ + key_len = key_len / 2; xts_ctx->key_len = key_len; + memcpy(xts_ctx->key, in_key, key_len); + memcpy(xts_ctx->pcc_key, in_key + key_len, key_len); return 0; } -static int xts_aes_crypt(struct blkcipher_desc *desc, long func, - struct s390_xts_ctx *xts_ctx, +static int xts_aes_crypt(struct blkcipher_desc *desc, unsigned long modifier, struct blkcipher_walk *walk) { - unsigned int offset = (xts_ctx->key_len >> 1) & 0x10; - int ret = blkcipher_walk_virt(desc, walk); - unsigned int nbytes = walk->nbytes; - unsigned int n; - u8 *in, *out; - struct pcc_param pcc_param; + struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int offset, nbytes, n; + int ret; + struct { + u8 key[32]; + u8 tweak[16]; + u8 block[16]; + u8 bit[16]; + u8 xts[16]; + } pcc_param; struct { u8 key[32]; u8 init[16]; } xts_param; - if (!nbytes) - goto out; - + ret = blkcipher_walk_virt(desc, walk); + offset = xts_ctx->key_len & 0x10; memset(pcc_param.block, 0, sizeof(pcc_param.block)); memset(pcc_param.bit, 0, sizeof(pcc_param.bit)); memset(pcc_param.xts, 0, sizeof(pcc_param.xts)); memcpy(pcc_param.tweak, walk->iv, sizeof(pcc_param.tweak)); - memcpy(pcc_param.key, xts_ctx->pcc_key, 32); - /* remove decipher modifier bit from 'func' and call PCC */ - ret = cpacf_pcc(func & 0x7f, &pcc_param.key[offset]); - if (ret < 0) - return -EIO; + memcpy(pcc_param.key + offset, xts_ctx->pcc_key, xts_ctx->key_len); + cpacf_pcc(xts_ctx->fc, pcc_param.key + offset); - memcpy(xts_param.key, xts_ctx->key, 32); + memcpy(xts_param.key + offset, xts_ctx->key, xts_ctx->key_len); memcpy(xts_param.init, pcc_param.xts, 16); - do { + + while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); - out = walk->dst.virt.addr; - in = walk->src.virt.addr; - - ret = cpacf_km(func, &xts_param.key[offset], out, in, n); - if (ret < 0 || ret != n) - return -EIO; - - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, walk, nbytes); - } while ((nbytes = walk->nbytes)); -out: + cpacf_km(xts_ctx->fc | modifier, xts_param.key + offset, + walk->dst.virt.addr, walk->src.virt.addr, n); + ret = blkcipher_walk_done(desc, walk, nbytes - n); + } return ret; } @@ -681,11 +565,11 @@ static int xts_aes_encrypt(struct blkcipher_desc *desc, struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; - if (unlikely(xts_ctx->key_len == 48)) + if (unlikely(!xts_ctx->fc)) return xts_fallback_encrypt(desc, dst, src, nbytes); blkcipher_walk_init(&walk, dst, src, nbytes); - return xts_aes_crypt(desc, xts_ctx->enc, xts_ctx, &walk); + return xts_aes_crypt(desc, 0, &walk); } static int xts_aes_decrypt(struct blkcipher_desc *desc, @@ -695,11 +579,11 @@ static int xts_aes_decrypt(struct blkcipher_desc *desc, struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; - if (unlikely(xts_ctx->key_len == 48)) + if (unlikely(!xts_ctx->fc)) return xts_fallback_decrypt(desc, dst, src, nbytes); blkcipher_walk_init(&walk, dst, src, nbytes); - return xts_aes_crypt(desc, xts_ctx->dec, xts_ctx, &walk); + return xts_aes_crypt(desc, CPACF_DECRYPT, &walk); } static int xts_fallback_init(struct crypto_tfm *tfm) @@ -750,108 +634,79 @@ static struct crypto_alg xts_aes_alg = { } }; -static int xts_aes_alg_reg; - static int ctr_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) { struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); + unsigned long fc; - switch (key_len) { - case 16: - sctx->enc = CPACF_KMCTR_AES_128_ENC; - sctx->dec = CPACF_KMCTR_AES_128_DEC; - break; - case 24: - sctx->enc = CPACF_KMCTR_AES_192_ENC; - sctx->dec = CPACF_KMCTR_AES_192_DEC; - break; - case 32: - sctx->enc = CPACF_KMCTR_AES_256_ENC; - sctx->dec = CPACF_KMCTR_AES_256_DEC; - break; - } + /* Pick the correct function code based on the key length */ + fc = (key_len == 16) ? CPACF_KMCTR_AES_128 : + (key_len == 24) ? CPACF_KMCTR_AES_192 : + (key_len == 32) ? CPACF_KMCTR_AES_256 : 0; + + /* Check if the function code is available */ + sctx->fc = (fc && cpacf_test_func(&kmctr_functions, fc)) ? fc : 0; + if (!sctx->fc) + return setkey_fallback_blk(tfm, in_key, key_len); - return aes_set_key(tfm, in_key, key_len); + sctx->key_len = key_len; + memcpy(sctx->key, in_key, key_len); + return 0; } -static unsigned int __ctrblk_init(u8 *ctrptr, unsigned int nbytes) +static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) { unsigned int i, n; /* only use complete blocks, max. PAGE_SIZE */ + memcpy(ctrptr, iv, AES_BLOCK_SIZE); n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1); - for (i = AES_BLOCK_SIZE; i < n; i += AES_BLOCK_SIZE) { - memcpy(ctrptr + i, ctrptr + i - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - crypto_inc(ctrptr + i, AES_BLOCK_SIZE); + for (i = (n / AES_BLOCK_SIZE) - 1; i > 0; i--) { + memcpy(ctrptr + AES_BLOCK_SIZE, ctrptr, AES_BLOCK_SIZE); + crypto_inc(ctrptr + AES_BLOCK_SIZE, AES_BLOCK_SIZE); + ctrptr += AES_BLOCK_SIZE; } return n; } -static int ctr_aes_crypt(struct blkcipher_desc *desc, long func, - struct s390_aes_ctx *sctx, struct blkcipher_walk *walk) +static int ctr_aes_crypt(struct blkcipher_desc *desc, unsigned long modifier, + struct blkcipher_walk *walk) { - int ret = blkcipher_walk_virt_block(desc, walk, AES_BLOCK_SIZE); + struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + u8 buf[AES_BLOCK_SIZE], *ctrptr; unsigned int n, nbytes; - u8 buf[AES_BLOCK_SIZE], ctrbuf[AES_BLOCK_SIZE]; - u8 *out, *in, *ctrptr = ctrbuf; - - if (!walk->nbytes) - return ret; + int ret, locked; - if (spin_trylock(&ctrblk_lock)) - ctrptr = ctrblk; + locked = spin_trylock(&ctrblk_lock); - memcpy(ctrptr, walk->iv, AES_BLOCK_SIZE); + ret = blkcipher_walk_virt_block(desc, walk, AES_BLOCK_SIZE); while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { - out = walk->dst.virt.addr; - in = walk->src.virt.addr; - while (nbytes >= AES_BLOCK_SIZE) { - if (ctrptr == ctrblk) - n = __ctrblk_init(ctrptr, nbytes); - else - n = AES_BLOCK_SIZE; - ret = cpacf_kmctr(func, sctx->key, out, in, n, ctrptr); - if (ret < 0 || ret != n) { - if (ctrptr == ctrblk) - spin_unlock(&ctrblk_lock); - return -EIO; - } - if (n > AES_BLOCK_SIZE) - memcpy(ctrptr, ctrptr + n - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - crypto_inc(ctrptr, AES_BLOCK_SIZE); - out += n; - in += n; - nbytes -= n; - } - ret = blkcipher_walk_done(desc, walk, nbytes); + n = AES_BLOCK_SIZE; + if (nbytes >= 2*AES_BLOCK_SIZE && locked) + n = __ctrblk_init(ctrblk, walk->iv, nbytes); + ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; + cpacf_kmctr(sctx->fc | modifier, sctx->key, + walk->dst.virt.addr, walk->src.virt.addr, + n, ctrptr); + if (ctrptr == ctrblk) + memcpy(walk->iv, ctrptr + n - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(walk->iv, AES_BLOCK_SIZE); + ret = blkcipher_walk_done(desc, walk, nbytes - n); } - if (ctrptr == ctrblk) { - if (nbytes) - memcpy(ctrbuf, ctrptr, AES_BLOCK_SIZE); - else - memcpy(walk->iv, ctrptr, AES_BLOCK_SIZE); + if (locked) spin_unlock(&ctrblk_lock); - } else { - if (!nbytes) - memcpy(walk->iv, ctrptr, AES_BLOCK_SIZE); - } /* * final block may be < AES_BLOCK_SIZE, copy only nbytes */ if (nbytes) { - out = walk->dst.virt.addr; - in = walk->src.virt.addr; - ret = cpacf_kmctr(func, sctx->key, buf, in, - AES_BLOCK_SIZE, ctrbuf); - if (ret < 0 || ret != AES_BLOCK_SIZE) - return -EIO; - memcpy(out, buf, nbytes); - crypto_inc(ctrbuf, AES_BLOCK_SIZE); + cpacf_kmctr(sctx->fc | modifier, sctx->key, + buf, walk->src.virt.addr, + AES_BLOCK_SIZE, walk->iv); + memcpy(walk->dst.virt.addr, buf, nbytes); + crypto_inc(walk->iv, AES_BLOCK_SIZE); ret = blkcipher_walk_done(desc, walk, 0); - memcpy(walk->iv, ctrbuf, AES_BLOCK_SIZE); } return ret; @@ -864,8 +719,11 @@ static int ctr_aes_encrypt(struct blkcipher_desc *desc, struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; + if (unlikely(!sctx->fc)) + return fallback_blk_enc(desc, dst, src, nbytes); + blkcipher_walk_init(&walk, dst, src, nbytes); - return ctr_aes_crypt(desc, sctx->enc, sctx, &walk); + return ctr_aes_crypt(desc, 0, &walk); } static int ctr_aes_decrypt(struct blkcipher_desc *desc, @@ -875,19 +733,25 @@ static int ctr_aes_decrypt(struct blkcipher_desc *desc, struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; + if (unlikely(!sctx->fc)) + return fallback_blk_dec(desc, dst, src, nbytes); + blkcipher_walk_init(&walk, dst, src, nbytes); - return ctr_aes_crypt(desc, sctx->dec, sctx, &walk); + return ctr_aes_crypt(desc, CPACF_DECRYPT, &walk); } static struct crypto_alg ctr_aes_alg = { .cra_name = "ctr(aes)", .cra_driver_name = "ctr-aes-s390", .cra_priority = 400, /* combo: aes + ctr */ - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct s390_aes_ctx), .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, + .cra_init = fallback_init_blk, + .cra_exit = fallback_exit_blk, .cra_u = { .blkcipher = { .min_keysize = AES_MIN_KEY_SIZE, @@ -900,89 +764,79 @@ static struct crypto_alg ctr_aes_alg = { } }; -static int ctr_aes_alg_reg; +static struct crypto_alg *aes_s390_algs_ptr[5]; +static int aes_s390_algs_num; -static int __init aes_s390_init(void) +static int aes_s390_register_alg(struct crypto_alg *alg) { int ret; - if (cpacf_query(CPACF_KM, CPACF_KM_AES_128_ENC)) - keylen_flag |= AES_KEYLEN_128; - if (cpacf_query(CPACF_KM, CPACF_KM_AES_192_ENC)) - keylen_flag |= AES_KEYLEN_192; - if (cpacf_query(CPACF_KM, CPACF_KM_AES_256_ENC)) - keylen_flag |= AES_KEYLEN_256; + ret = crypto_register_alg(alg); + if (!ret) + aes_s390_algs_ptr[aes_s390_algs_num++] = alg; + return ret; +} - if (!keylen_flag) - return -EOPNOTSUPP; +static void aes_s390_fini(void) +{ + while (aes_s390_algs_num--) + crypto_unregister_alg(aes_s390_algs_ptr[aes_s390_algs_num]); + if (ctrblk) + free_page((unsigned long) ctrblk); +} - /* z9 109 and z9 BC/EC only support 128 bit key length */ - if (keylen_flag == AES_KEYLEN_128) - pr_info("AES hardware acceleration is only available for" - " 128-bit keys\n"); +static int __init aes_s390_init(void) +{ + int ret; - ret = crypto_register_alg(&aes_alg); - if (ret) - goto aes_err; + /* Query available functions for KM, KMC and KMCTR */ + cpacf_query(CPACF_KM, &km_functions); + cpacf_query(CPACF_KMC, &kmc_functions); + cpacf_query(CPACF_KMCTR, &kmctr_functions); - ret = crypto_register_alg(&ecb_aes_alg); - if (ret) - goto ecb_aes_err; + if (cpacf_test_func(&km_functions, CPACF_KM_AES_128) || + cpacf_test_func(&km_functions, CPACF_KM_AES_192) || + cpacf_test_func(&km_functions, CPACF_KM_AES_256)) { + ret = aes_s390_register_alg(&aes_alg); + if (ret) + goto out_err; + ret = aes_s390_register_alg(&ecb_aes_alg); + if (ret) + goto out_err; + } - ret = crypto_register_alg(&cbc_aes_alg); - if (ret) - goto cbc_aes_err; + if (cpacf_test_func(&kmc_functions, CPACF_KMC_AES_128) || + cpacf_test_func(&kmc_functions, CPACF_KMC_AES_192) || + cpacf_test_func(&kmc_functions, CPACF_KMC_AES_256)) { + ret = aes_s390_register_alg(&cbc_aes_alg); + if (ret) + goto out_err; + } - if (cpacf_query(CPACF_KM, CPACF_KM_XTS_128_ENC) && - cpacf_query(CPACF_KM, CPACF_KM_XTS_256_ENC)) { - ret = crypto_register_alg(&xts_aes_alg); + if (cpacf_test_func(&km_functions, CPACF_KM_XTS_128) || + cpacf_test_func(&km_functions, CPACF_KM_XTS_256)) { + ret = aes_s390_register_alg(&xts_aes_alg); if (ret) - goto xts_aes_err; - xts_aes_alg_reg = 1; + goto out_err; } - if (cpacf_query(CPACF_KMCTR, CPACF_KMCTR_AES_128_ENC) && - cpacf_query(CPACF_KMCTR, CPACF_KMCTR_AES_192_ENC) && - cpacf_query(CPACF_KMCTR, CPACF_KMCTR_AES_256_ENC)) { + if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_AES_128) || + cpacf_test_func(&kmctr_functions, CPACF_KMCTR_AES_192) || + cpacf_test_func(&kmctr_functions, CPACF_KMCTR_AES_256)) { ctrblk = (u8 *) __get_free_page(GFP_KERNEL); if (!ctrblk) { ret = -ENOMEM; - goto ctr_aes_err; + goto out_err; } - ret = crypto_register_alg(&ctr_aes_alg); - if (ret) { - free_page((unsigned long) ctrblk); - goto ctr_aes_err; - } - ctr_aes_alg_reg = 1; + ret = aes_s390_register_alg(&ctr_aes_alg); + if (ret) + goto out_err; } -out: + return 0; +out_err: + aes_s390_fini(); return ret; - -ctr_aes_err: - crypto_unregister_alg(&xts_aes_alg); -xts_aes_err: - crypto_unregister_alg(&cbc_aes_alg); -cbc_aes_err: - crypto_unregister_alg(&ecb_aes_alg); -ecb_aes_err: - crypto_unregister_alg(&aes_alg); -aes_err: - goto out; -} - -static void __exit aes_s390_fini(void) -{ - if (ctr_aes_alg_reg) { - crypto_unregister_alg(&ctr_aes_alg); - free_page((unsigned long) ctrblk); - } - if (xts_aes_alg_reg) - crypto_unregister_alg(&xts_aes_alg); - crypto_unregister_alg(&cbc_aes_alg); - crypto_unregister_alg(&ecb_aes_alg); - crypto_unregister_alg(&aes_alg); } module_cpu_feature_match(MSA, aes_s390_init); diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c index 2bad9d837029..992e630c227b 100644 --- a/arch/s390/crypto/crc32-vx.c +++ b/arch/s390/crypto/crc32-vx.c @@ -67,7 +67,7 @@ u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); \ kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \ crc = ___crc32_vx(crc, data, aligned); \ - kernel_fpu_end(&vxstate); \ + kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \ \ if (remaining) \ crc = ___crc32_sw(crc, data + aligned, remaining); \ diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index 697e71a75fc2..8b83144206eb 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c @@ -27,6 +27,8 @@ static u8 *ctrblk; static DEFINE_SPINLOCK(ctrblk_lock); +static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; + struct s390_des_ctx { u8 iv[DES_BLOCK_SIZE]; u8 key[DES3_KEY_SIZE]; @@ -36,12 +38,12 @@ static int des_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key_len) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - u32 *flags = &tfm->crt_flags; u32 tmp[DES_EXPKEY_WORDS]; /* check for weak keys */ - if (!des_ekey(tmp, key) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; + if (!des_ekey(tmp, key) && + (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } @@ -53,14 +55,15 @@ static void des_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - cpacf_km(CPACF_KM_DEA_ENC, ctx->key, out, in, DES_BLOCK_SIZE); + cpacf_km(CPACF_KM_DEA, ctx->key, out, in, DES_BLOCK_SIZE); } static void des_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - cpacf_km(CPACF_KM_DEA_DEC, ctx->key, out, in, DES_BLOCK_SIZE); + cpacf_km(CPACF_KM_DEA | CPACF_DECRYPT, + ctx->key, out, in, DES_BLOCK_SIZE); } static struct crypto_alg des_alg = { @@ -82,61 +85,46 @@ static struct crypto_alg des_alg = { } }; -static int ecb_desall_crypt(struct blkcipher_desc *desc, long func, - u8 *key, struct blkcipher_walk *walk) +static int ecb_desall_crypt(struct blkcipher_desc *desc, unsigned long fc, + struct blkcipher_walk *walk) { - int ret = blkcipher_walk_virt(desc, walk); - unsigned int nbytes; + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int nbytes, n; + int ret; - while ((nbytes = walk->nbytes)) { + ret = blkcipher_walk_virt(desc, walk); + while ((nbytes = walk->nbytes) >= DES_BLOCK_SIZE) { /* only use complete blocks */ - unsigned int n = nbytes & ~(DES_BLOCK_SIZE - 1); - u8 *out = walk->dst.virt.addr; - u8 *in = walk->src.virt.addr; - - ret = cpacf_km(func, key, out, in, n); - if (ret < 0 || ret != n) - return -EIO; - - nbytes &= DES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, walk, nbytes); + n = nbytes & ~(DES_BLOCK_SIZE - 1); + cpacf_km(fc, ctx->key, walk->dst.virt.addr, + walk->src.virt.addr, n); + ret = blkcipher_walk_done(desc, walk, nbytes - n); } - return ret; } -static int cbc_desall_crypt(struct blkcipher_desc *desc, long func, +static int cbc_desall_crypt(struct blkcipher_desc *desc, unsigned long fc, struct blkcipher_walk *walk) { struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - int ret = blkcipher_walk_virt(desc, walk); - unsigned int nbytes = walk->nbytes; + unsigned int nbytes, n; + int ret; struct { u8 iv[DES_BLOCK_SIZE]; u8 key[DES3_KEY_SIZE]; } param; - if (!nbytes) - goto out; - + ret = blkcipher_walk_virt(desc, walk); memcpy(param.iv, walk->iv, DES_BLOCK_SIZE); memcpy(param.key, ctx->key, DES3_KEY_SIZE); - do { + while ((nbytes = walk->nbytes) >= DES_BLOCK_SIZE) { /* only use complete blocks */ - unsigned int n = nbytes & ~(DES_BLOCK_SIZE - 1); - u8 *out = walk->dst.virt.addr; - u8 *in = walk->src.virt.addr; - - ret = cpacf_kmc(func, ¶m, out, in, n); - if (ret < 0 || ret != n) - return -EIO; - - nbytes &= DES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, walk, nbytes); - } while ((nbytes = walk->nbytes)); + n = nbytes & ~(DES_BLOCK_SIZE - 1); + cpacf_kmc(fc, ¶m, walk->dst.virt.addr, + walk->src.virt.addr, n); + ret = blkcipher_walk_done(desc, walk, nbytes - n); + } memcpy(walk->iv, param.iv, DES_BLOCK_SIZE); - -out: return ret; } @@ -144,22 +132,20 @@ static int ecb_des_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, CPACF_KM_DEA_ENC, ctx->key, &walk); + return ecb_desall_crypt(desc, CPACF_KM_DEA, &walk); } static int ecb_des_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, CPACF_KM_DEA_DEC, ctx->key, &walk); + return ecb_desall_crypt(desc, CPACF_KM_DEA | CPACF_DECRYPT, &walk); } static struct crypto_alg ecb_des_alg = { @@ -189,7 +175,7 @@ static int cbc_des_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, CPACF_KMC_DEA_ENC, &walk); + return cbc_desall_crypt(desc, CPACF_KMC_DEA, &walk); } static int cbc_des_decrypt(struct blkcipher_desc *desc, @@ -199,7 +185,7 @@ static int cbc_des_decrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, CPACF_KMC_DEA_DEC, &walk); + return cbc_desall_crypt(desc, CPACF_KMC_DEA | CPACF_DECRYPT, &walk); } static struct crypto_alg cbc_des_alg = { @@ -240,13 +226,12 @@ static int des3_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key_len) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - u32 *flags = &tfm->crt_flags; if (!(crypto_memneq(key, &key[DES_KEY_SIZE], DES_KEY_SIZE) && crypto_memneq(&key[DES_KEY_SIZE], &key[DES_KEY_SIZE * 2], DES_KEY_SIZE)) && - (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; + (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } memcpy(ctx->key, key, key_len); @@ -257,14 +242,15 @@ static void des3_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - cpacf_km(CPACF_KM_TDEA_192_ENC, ctx->key, dst, src, DES_BLOCK_SIZE); + cpacf_km(CPACF_KM_TDEA_192, ctx->key, dst, src, DES_BLOCK_SIZE); } static void des3_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - cpacf_km(CPACF_KM_TDEA_192_DEC, ctx->key, dst, src, DES_BLOCK_SIZE); + cpacf_km(CPACF_KM_TDEA_192 | CPACF_DECRYPT, + ctx->key, dst, src, DES_BLOCK_SIZE); } static struct crypto_alg des3_alg = { @@ -290,22 +276,21 @@ static int ecb_des3_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, CPACF_KM_TDEA_192_ENC, ctx->key, &walk); + return ecb_desall_crypt(desc, CPACF_KM_TDEA_192, &walk); } static int ecb_des3_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, CPACF_KM_TDEA_192_DEC, ctx->key, &walk); + return ecb_desall_crypt(desc, CPACF_KM_TDEA_192 | CPACF_DECRYPT, + &walk); } static struct crypto_alg ecb_des3_alg = { @@ -335,7 +320,7 @@ static int cbc_des3_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, CPACF_KMC_TDEA_192_ENC, &walk); + return cbc_desall_crypt(desc, CPACF_KMC_TDEA_192, &walk); } static int cbc_des3_decrypt(struct blkcipher_desc *desc, @@ -345,7 +330,8 @@ static int cbc_des3_decrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, CPACF_KMC_TDEA_192_DEC, &walk); + return cbc_desall_crypt(desc, CPACF_KMC_TDEA_192 | CPACF_DECRYPT, + &walk); } static struct crypto_alg cbc_des3_alg = { @@ -369,81 +355,54 @@ static struct crypto_alg cbc_des3_alg = { } }; -static unsigned int __ctrblk_init(u8 *ctrptr, unsigned int nbytes) +static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) { unsigned int i, n; /* align to block size, max. PAGE_SIZE */ n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(DES_BLOCK_SIZE - 1); - for (i = DES_BLOCK_SIZE; i < n; i += DES_BLOCK_SIZE) { - memcpy(ctrptr + i, ctrptr + i - DES_BLOCK_SIZE, DES_BLOCK_SIZE); - crypto_inc(ctrptr + i, DES_BLOCK_SIZE); + memcpy(ctrptr, iv, DES_BLOCK_SIZE); + for (i = (n / DES_BLOCK_SIZE) - 1; i > 0; i--) { + memcpy(ctrptr + DES_BLOCK_SIZE, ctrptr, DES_BLOCK_SIZE); + crypto_inc(ctrptr + DES_BLOCK_SIZE, DES_BLOCK_SIZE); + ctrptr += DES_BLOCK_SIZE; } return n; } -static int ctr_desall_crypt(struct blkcipher_desc *desc, long func, - struct s390_des_ctx *ctx, +static int ctr_desall_crypt(struct blkcipher_desc *desc, unsigned long fc, struct blkcipher_walk *walk) { - int ret = blkcipher_walk_virt_block(desc, walk, DES_BLOCK_SIZE); + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + u8 buf[DES_BLOCK_SIZE], *ctrptr; unsigned int n, nbytes; - u8 buf[DES_BLOCK_SIZE], ctrbuf[DES_BLOCK_SIZE]; - u8 *out, *in, *ctrptr = ctrbuf; - - if (!walk->nbytes) - return ret; + int ret, locked; - if (spin_trylock(&ctrblk_lock)) - ctrptr = ctrblk; + locked = spin_trylock(&ctrblk_lock); - memcpy(ctrptr, walk->iv, DES_BLOCK_SIZE); + ret = blkcipher_walk_virt_block(desc, walk, DES_BLOCK_SIZE); while ((nbytes = walk->nbytes) >= DES_BLOCK_SIZE) { - out = walk->dst.virt.addr; - in = walk->src.virt.addr; - while (nbytes >= DES_BLOCK_SIZE) { - if (ctrptr == ctrblk) - n = __ctrblk_init(ctrptr, nbytes); - else - n = DES_BLOCK_SIZE; - ret = cpacf_kmctr(func, ctx->key, out, in, n, ctrptr); - if (ret < 0 || ret != n) { - if (ctrptr == ctrblk) - spin_unlock(&ctrblk_lock); - return -EIO; - } - if (n > DES_BLOCK_SIZE) - memcpy(ctrptr, ctrptr + n - DES_BLOCK_SIZE, - DES_BLOCK_SIZE); - crypto_inc(ctrptr, DES_BLOCK_SIZE); - out += n; - in += n; - nbytes -= n; - } - ret = blkcipher_walk_done(desc, walk, nbytes); + n = DES_BLOCK_SIZE; + if (nbytes >= 2*DES_BLOCK_SIZE && locked) + n = __ctrblk_init(ctrblk, walk->iv, nbytes); + ctrptr = (n > DES_BLOCK_SIZE) ? ctrblk : walk->iv; + cpacf_kmctr(fc, ctx->key, walk->dst.virt.addr, + walk->src.virt.addr, n, ctrptr); + if (ctrptr == ctrblk) + memcpy(walk->iv, ctrptr + n - DES_BLOCK_SIZE, + DES_BLOCK_SIZE); + crypto_inc(walk->iv, DES_BLOCK_SIZE); + ret = blkcipher_walk_done(desc, walk, nbytes - n); } - if (ctrptr == ctrblk) { - if (nbytes) - memcpy(ctrbuf, ctrptr, DES_BLOCK_SIZE); - else - memcpy(walk->iv, ctrptr, DES_BLOCK_SIZE); + if (locked) spin_unlock(&ctrblk_lock); - } else { - if (!nbytes) - memcpy(walk->iv, ctrptr, DES_BLOCK_SIZE); - } /* final block may be < DES_BLOCK_SIZE, copy only nbytes */ if (nbytes) { - out = walk->dst.virt.addr; - in = walk->src.virt.addr; - ret = cpacf_kmctr(func, ctx->key, buf, in, - DES_BLOCK_SIZE, ctrbuf); - if (ret < 0 || ret != DES_BLOCK_SIZE) - return -EIO; - memcpy(out, buf, nbytes); - crypto_inc(ctrbuf, DES_BLOCK_SIZE); + cpacf_kmctr(fc, ctx->key, buf, walk->src.virt.addr, + DES_BLOCK_SIZE, walk->iv); + memcpy(walk->dst.virt.addr, buf, nbytes); + crypto_inc(walk->iv, DES_BLOCK_SIZE); ret = blkcipher_walk_done(desc, walk, 0); - memcpy(walk->iv, ctrbuf, DES_BLOCK_SIZE); } return ret; } @@ -452,22 +411,20 @@ static int ctr_des_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ctr_desall_crypt(desc, CPACF_KMCTR_DEA_ENC, ctx, &walk); + return ctr_desall_crypt(desc, CPACF_KMCTR_DEA, &walk); } static int ctr_des_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ctr_desall_crypt(desc, CPACF_KMCTR_DEA_DEC, ctx, &walk); + return ctr_desall_crypt(desc, CPACF_KMCTR_DEA | CPACF_DECRYPT, &walk); } static struct crypto_alg ctr_des_alg = { @@ -495,22 +452,21 @@ static int ctr_des3_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ctr_desall_crypt(desc, CPACF_KMCTR_TDEA_192_ENC, ctx, &walk); + return ctr_desall_crypt(desc, CPACF_KMCTR_TDEA_192, &walk); } static int ctr_des3_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ctr_desall_crypt(desc, CPACF_KMCTR_TDEA_192_DEC, ctx, &walk); + return ctr_desall_crypt(desc, CPACF_KMCTR_TDEA_192 | CPACF_DECRYPT, + &walk); } static struct crypto_alg ctr_des3_alg = { @@ -534,83 +490,87 @@ static struct crypto_alg ctr_des3_alg = { } }; +static struct crypto_alg *des_s390_algs_ptr[8]; +static int des_s390_algs_num; + +static int des_s390_register_alg(struct crypto_alg *alg) +{ + int ret; + + ret = crypto_register_alg(alg); + if (!ret) + des_s390_algs_ptr[des_s390_algs_num++] = alg; + return ret; +} + +static void des_s390_exit(void) +{ + while (des_s390_algs_num--) + crypto_unregister_alg(des_s390_algs_ptr[des_s390_algs_num]); + if (ctrblk) + free_page((unsigned long) ctrblk); +} + static int __init des_s390_init(void) { int ret; - if (!cpacf_query(CPACF_KM, CPACF_KM_DEA_ENC) || - !cpacf_query(CPACF_KM, CPACF_KM_TDEA_192_ENC)) - return -EOPNOTSUPP; - - ret = crypto_register_alg(&des_alg); - if (ret) - goto des_err; - ret = crypto_register_alg(&ecb_des_alg); - if (ret) - goto ecb_des_err; - ret = crypto_register_alg(&cbc_des_alg); - if (ret) - goto cbc_des_err; - ret = crypto_register_alg(&des3_alg); - if (ret) - goto des3_err; - ret = crypto_register_alg(&ecb_des3_alg); - if (ret) - goto ecb_des3_err; - ret = crypto_register_alg(&cbc_des3_alg); - if (ret) - goto cbc_des3_err; - - if (cpacf_query(CPACF_KMCTR, CPACF_KMCTR_DEA_ENC) && - cpacf_query(CPACF_KMCTR, CPACF_KMCTR_TDEA_192_ENC)) { - ret = crypto_register_alg(&ctr_des_alg); + /* Query available functions for KM, KMC and KMCTR */ + cpacf_query(CPACF_KM, &km_functions); + cpacf_query(CPACF_KMC, &kmc_functions); + cpacf_query(CPACF_KMCTR, &kmctr_functions); + + if (cpacf_test_func(&km_functions, CPACF_KM_DEA)) { + ret = des_s390_register_alg(&des_alg); + if (ret) + goto out_err; + ret = des_s390_register_alg(&ecb_des_alg); if (ret) - goto ctr_des_err; - ret = crypto_register_alg(&ctr_des3_alg); + goto out_err; + } + if (cpacf_test_func(&kmc_functions, CPACF_KMC_DEA)) { + ret = des_s390_register_alg(&cbc_des_alg); if (ret) - goto ctr_des3_err; + goto out_err; + } + if (cpacf_test_func(&km_functions, CPACF_KM_TDEA_192)) { + ret = des_s390_register_alg(&des3_alg); + if (ret) + goto out_err; + ret = des_s390_register_alg(&ecb_des3_alg); + if (ret) + goto out_err; + } + if (cpacf_test_func(&kmc_functions, CPACF_KMC_TDEA_192)) { + ret = des_s390_register_alg(&cbc_des3_alg); + if (ret) + goto out_err; + } + + if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_DEA) || + cpacf_test_func(&kmctr_functions, CPACF_KMCTR_TDEA_192)) { ctrblk = (u8 *) __get_free_page(GFP_KERNEL); if (!ctrblk) { ret = -ENOMEM; - goto ctr_mem_err; + goto out_err; } } -out: - return ret; -ctr_mem_err: - crypto_unregister_alg(&ctr_des3_alg); -ctr_des3_err: - crypto_unregister_alg(&ctr_des_alg); -ctr_des_err: - crypto_unregister_alg(&cbc_des3_alg); -cbc_des3_err: - crypto_unregister_alg(&ecb_des3_alg); -ecb_des3_err: - crypto_unregister_alg(&des3_alg); -des3_err: - crypto_unregister_alg(&cbc_des_alg); -cbc_des_err: - crypto_unregister_alg(&ecb_des_alg); -ecb_des_err: - crypto_unregister_alg(&des_alg); -des_err: - goto out; -} - -static void __exit des_s390_exit(void) -{ - if (ctrblk) { - crypto_unregister_alg(&ctr_des_alg); - crypto_unregister_alg(&ctr_des3_alg); - free_page((unsigned long) ctrblk); + if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_DEA)) { + ret = des_s390_register_alg(&ctr_des_alg); + if (ret) + goto out_err; + } + if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_TDEA_192)) { + ret = des_s390_register_alg(&ctr_des3_alg); + if (ret) + goto out_err; } - crypto_unregister_alg(&cbc_des3_alg); - crypto_unregister_alg(&ecb_des3_alg); - crypto_unregister_alg(&des3_alg); - crypto_unregister_alg(&cbc_des_alg); - crypto_unregister_alg(&ecb_des_alg); - crypto_unregister_alg(&des_alg); + + return 0; +out_err: + des_s390_exit(); + return ret; } module_cpu_feature_match(MSA, des_s390_init); diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c index ab68de72e795..564616d48d8b 100644 --- a/arch/s390/crypto/ghash_s390.c +++ b/arch/s390/crypto/ghash_s390.c @@ -58,7 +58,6 @@ static int ghash_update(struct shash_desc *desc, struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); unsigned int n; u8 *buf = dctx->buffer; - int ret; if (dctx->bytes) { u8 *pos = buf + (GHASH_BLOCK_SIZE - dctx->bytes); @@ -71,18 +70,14 @@ static int ghash_update(struct shash_desc *desc, src += n; if (!dctx->bytes) { - ret = cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, - GHASH_BLOCK_SIZE); - if (ret != GHASH_BLOCK_SIZE) - return -EIO; + cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, + GHASH_BLOCK_SIZE); } } n = srclen & ~(GHASH_BLOCK_SIZE - 1); if (n) { - ret = cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n); - if (ret != n) - return -EIO; + cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n); src += n; srclen -= n; } @@ -98,17 +93,12 @@ static int ghash_update(struct shash_desc *desc, static int ghash_flush(struct ghash_desc_ctx *dctx) { u8 *buf = dctx->buffer; - int ret; if (dctx->bytes) { u8 *pos = buf + (GHASH_BLOCK_SIZE - dctx->bytes); memset(pos, 0, dctx->bytes); - - ret = cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); - if (ret != GHASH_BLOCK_SIZE) - return -EIO; - + cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); dctx->bytes = 0; } @@ -146,7 +136,7 @@ static struct shash_alg ghash_alg = { static int __init ghash_mod_init(void) { - if (!cpacf_query(CPACF_KIMD, CPACF_KIMD_GHASH)) + if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH)) return -EOPNOTSUPP; return crypto_register_shash(&ghash_alg); diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c index 41527b113f5a..9cc050f9536c 100644 --- a/arch/s390/crypto/prng.c +++ b/arch/s390/crypto/prng.c @@ -135,12 +135,7 @@ static int generate_entropy(u8 *ebuf, size_t nbytes) else h = ebuf; /* generate sha256 from this page */ - if (cpacf_kimd(CPACF_KIMD_SHA_256, h, - pg, PAGE_SIZE) != PAGE_SIZE) { - prng_errorflag = PRNG_GEN_ENTROPY_FAILED; - ret = -EIO; - goto out; - } + cpacf_kimd(CPACF_KIMD_SHA_256, h, pg, PAGE_SIZE); if (n < sizeof(hash)) memcpy(ebuf, hash, n); ret += n; @@ -148,7 +143,6 @@ static int generate_entropy(u8 *ebuf, size_t nbytes) nbytes -= n; } -out: free_page((unsigned long)pg); return ret; } @@ -160,13 +154,11 @@ static void prng_tdes_add_entropy(void) { __u64 entropy[4]; unsigned int i; - int ret; for (i = 0; i < 16; i++) { - ret = cpacf_kmc(CPACF_KMC_PRNG, prng_data->prngws.parm_block, - (char *)entropy, (char *)entropy, - sizeof(entropy)); - BUG_ON(ret < 0 || ret != sizeof(entropy)); + cpacf_kmc(CPACF_KMC_PRNG, prng_data->prngws.parm_block, + (char *) entropy, (char *) entropy, + sizeof(entropy)); memcpy(prng_data->prngws.parm_block, entropy, sizeof(entropy)); } } @@ -303,21 +295,14 @@ static int __init prng_sha512_selftest(void) 0x21, 0xe4, 0xb0, 0x86, 0x44, 0xf6, 0x72, 0x7c, 0x36, 0x8c, 0x5a, 0x9f, 0x7a, 0x4b, 0x3e, 0xe2 }; - int ret = 0; u8 buf[sizeof(random)]; struct ppno_ws_s ws; memset(&ws, 0, sizeof(ws)); /* initial seed */ - ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_SEED, &ws, NULL, 0, - seed, sizeof(seed)); - if (ret < 0) { - pr_err("The prng self test seed operation for the " - "SHA-512 mode failed with rc=%d\n", ret); - prng_errorflag = PRNG_SELFTEST_FAILED; - return -EIO; - } + cpacf_ppno(CPACF_PPNO_SHA512_DRNG_SEED, + &ws, NULL, 0, seed, sizeof(seed)); /* check working states V and C */ if (memcmp(ws.V, V0, sizeof(V0)) != 0 @@ -329,22 +314,10 @@ static int __init prng_sha512_selftest(void) } /* generate random bytes */ - ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, - &ws, buf, sizeof(buf), NULL, 0); - if (ret < 0) { - pr_err("The prng self test generate operation for " - "the SHA-512 mode failed with rc=%d\n", ret); - prng_errorflag = PRNG_SELFTEST_FAILED; - return -EIO; - } - ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, - &ws, buf, sizeof(buf), NULL, 0); - if (ret < 0) { - pr_err("The prng self test generate operation for " - "the SHA-512 mode failed with rc=%d\n", ret); - prng_errorflag = PRNG_SELFTEST_FAILED; - return -EIO; - } + cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, + &ws, buf, sizeof(buf), NULL, 0); + cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, + &ws, buf, sizeof(buf), NULL, 0); /* check against expected data */ if (memcmp(buf, random, sizeof(random)) != 0) { @@ -392,26 +365,16 @@ static int __init prng_sha512_instantiate(void) get_tod_clock_ext(seed + 48); /* initial seed of the ppno drng */ - ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_SEED, - &prng_data->ppnows, NULL, 0, seed, sizeof(seed)); - if (ret < 0) { - prng_errorflag = PRNG_SEED_FAILED; - ret = -EIO; - goto outfree; - } + cpacf_ppno(CPACF_PPNO_SHA512_DRNG_SEED, + &prng_data->ppnows, NULL, 0, seed, sizeof(seed)); /* if fips mode is enabled, generate a first block of random bytes for the FIPS 140-2 Conditional Self Test */ if (fips_enabled) { prng_data->prev = prng_data->buf + prng_chunk_size; - ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, - &prng_data->ppnows, - prng_data->prev, prng_chunk_size, NULL, 0); - if (ret < 0 || ret != prng_chunk_size) { - prng_errorflag = PRNG_GEN_FAILED; - ret = -EIO; - goto outfree; - } + cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, + &prng_data->ppnows, + prng_data->prev, prng_chunk_size, NULL, 0); } return 0; @@ -440,12 +403,8 @@ static int prng_sha512_reseed(void) return ret; /* do a reseed of the ppno drng with this bytestring */ - ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_SEED, - &prng_data->ppnows, NULL, 0, seed, sizeof(seed)); - if (ret) { - prng_errorflag = PRNG_RESEED_FAILED; - return -EIO; - } + cpacf_ppno(CPACF_PPNO_SHA512_DRNG_SEED, + &prng_data->ppnows, NULL, 0, seed, sizeof(seed)); return 0; } @@ -463,12 +422,8 @@ static int prng_sha512_generate(u8 *buf, size_t nbytes) } /* PPNO generate */ - ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, - &prng_data->ppnows, buf, nbytes, NULL, 0); - if (ret < 0 || ret != nbytes) { - prng_errorflag = PRNG_GEN_FAILED; - return -EIO; - } + cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, + &prng_data->ppnows, buf, nbytes, NULL, 0); /* FIPS 140-2 Conditional Self Test */ if (fips_enabled) { @@ -479,7 +434,7 @@ static int prng_sha512_generate(u8 *buf, size_t nbytes) memcpy(prng_data->prev, buf, nbytes); } - return ret; + return nbytes; } @@ -494,7 +449,7 @@ static int prng_open(struct inode *inode, struct file *file) static ssize_t prng_tdes_read(struct file *file, char __user *ubuf, size_t nbytes, loff_t *ppos) { - int chunk, n, tmp, ret = 0; + int chunk, n, ret = 0; /* lock prng_data struct */ if (mutex_lock_interruptible(&prng_data->mutex)) @@ -545,13 +500,9 @@ static ssize_t prng_tdes_read(struct file *file, char __user *ubuf, * * Note: you can still get strict X9.17 conformity by setting * prng_chunk_size to 8 bytes. - */ - tmp = cpacf_kmc(CPACF_KMC_PRNG, prng_data->prngws.parm_block, - prng_data->buf, prng_data->buf, n); - if (tmp < 0 || tmp != n) { - ret = -EIO; - break; - } + */ + cpacf_kmc(CPACF_KMC_PRNG, prng_data->prngws.parm_block, + prng_data->buf, prng_data->buf, n); prng_data->prngws.byte_counter += n; prng_data->prngws.reseed_counter += n; @@ -806,13 +757,13 @@ static int __init prng_init(void) int ret; /* check if the CPU has a PRNG */ - if (!cpacf_query(CPACF_KMC, CPACF_KMC_PRNG)) + if (!cpacf_query_func(CPACF_KMC, CPACF_KMC_PRNG)) return -EOPNOTSUPP; /* choose prng mode */ if (prng_mode != PRNG_MODE_TDES) { /* check for MSA5 support for PPNO operations */ - if (!cpacf_query(CPACF_PPNO, CPACF_PPNO_SHA512_DRNG_GEN)) { + if (!cpacf_query_func(CPACF_PPNO, CPACF_PPNO_SHA512_DRNG_GEN)) { if (prng_mode == PRNG_MODE_SHA512) { pr_err("The prng module cannot " "start in SHA-512 mode\n"); diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index 5fbf91bbb478..c7de53d8da75 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -91,7 +91,7 @@ static struct shash_alg alg = { static int __init sha1_s390_init(void) { - if (!cpacf_query(CPACF_KIMD, CPACF_KIMD_SHA_1)) + if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_1)) return -EOPNOTSUPP; return crypto_register_shash(&alg); } diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index 10aac0b11988..53c277999a28 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -123,7 +123,7 @@ static int __init sha256_s390_init(void) { int ret; - if (!cpacf_query(CPACF_KIMD, CPACF_KIMD_SHA_256)) + if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_256)) return -EOPNOTSUPP; ret = crypto_register_shash(&sha256_alg); if (ret < 0) diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index ea85757be407..2f4caa1ef123 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -133,7 +133,7 @@ static int __init init(void) { int ret; - if (!cpacf_query(CPACF_KIMD, CPACF_KIMD_SHA_512)) + if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_512)) return -EOPNOTSUPP; if ((ret = crypto_register_shash(&sha512_alg)) < 0) goto out; diff --git a/arch/s390/crypto/sha_common.c b/arch/s390/crypto/sha_common.c index 8e908166c3ee..c740f77285b2 100644 --- a/arch/s390/crypto/sha_common.c +++ b/arch/s390/crypto/sha_common.c @@ -22,8 +22,7 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct s390_sha_ctx *ctx = shash_desc_ctx(desc); unsigned int bsize = crypto_shash_blocksize(desc->tfm); - unsigned int index; - int ret; + unsigned int index, n; /* how much is already in the buffer? */ index = ctx->count & (bsize - 1); @@ -35,9 +34,7 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) /* process one stored block */ if (index) { memcpy(ctx->buf + index, data, bsize - index); - ret = cpacf_kimd(ctx->func, ctx->state, ctx->buf, bsize); - if (ret != bsize) - return -EIO; + cpacf_kimd(ctx->func, ctx->state, ctx->buf, bsize); data += bsize - index; len -= bsize - index; index = 0; @@ -45,12 +42,10 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) /* process as many blocks as possible */ if (len >= bsize) { - ret = cpacf_kimd(ctx->func, ctx->state, data, - len & ~(bsize - 1)); - if (ret != (len & ~(bsize - 1))) - return -EIO; - data += ret; - len -= ret; + n = len & ~(bsize - 1); + cpacf_kimd(ctx->func, ctx->state, data, n); + data += n; + len -= n; } store: if (len) @@ -66,7 +61,6 @@ int s390_sha_final(struct shash_desc *desc, u8 *out) unsigned int bsize = crypto_shash_blocksize(desc->tfm); u64 bits; unsigned int index, end, plen; - int ret; /* SHA-512 uses 128 bit padding length */ plen = (bsize > SHA256_BLOCK_SIZE) ? 16 : 8; @@ -88,10 +82,7 @@ int s390_sha_final(struct shash_desc *desc, u8 *out) */ bits = ctx->count * 8; memcpy(ctx->buf + end - 8, &bits, sizeof(bits)); - - ret = cpacf_kimd(ctx->func, ctx->state, ctx->buf, end); - if (ret != end) - return -EIO; + cpacf_kimd(ctx->func, ctx->state, ctx->buf, end); /* copy digest to out */ memcpy(out, ctx->state, crypto_shash_digestsize(desc->tfm)); |