From d4b3984c9e627eacbbf5ee81f9a60eb3191bff8e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 13 Oct 2020 16:26:50 -0700 Subject: crypto: x86/aes - remove unused file aes_glue.c Commit 1d2c3279311e ("crypto: x86/aes - drop scalar assembler implementations") was meant to remove aes_glue.c, but it actually left it as an unused one-line file. Remove this unused file. Cc: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/aes_glue.c | 1 - 1 file changed, 1 deletion(-) delete mode 100644 arch/x86/crypto/aes_glue.c (limited to 'arch') diff --git a/arch/x86/crypto/aes_glue.c b/arch/x86/crypto/aes_glue.c deleted file mode 100644 index 7b7dc05fa1a4..000000000000 --- a/arch/x86/crypto/aes_glue.c +++ /dev/null @@ -1 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -- cgit v1.2.3 From 458c0480dcb338d7b72e89b2e88a622965adcea4 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 25 Oct 2020 10:31:15 -0400 Subject: crypto: hash - Use memzero_explicit() for clearing state Without the barrier_data() inside memzero_explicit(), the compiler may optimize away the state-clearing if it can tell that the state is not used afterwards. Signed-off-by: Arvind Sankar Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/ghash-ce-glue.c | 2 +- arch/arm64/crypto/poly1305-glue.c | 2 +- arch/arm64/crypto/sha3-ce-glue.c | 2 +- arch/x86/crypto/poly1305_glue.c | 2 +- include/crypto/sha1_base.h | 3 ++- include/crypto/sha256_base.h | 3 ++- include/crypto/sha512_base.h | 3 ++- include/crypto/sm3_base.h | 3 ++- 8 files changed, 12 insertions(+), 8 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 8536008e3e35..2427e2f3a9a1 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -168,7 +168,7 @@ static int ghash_final(struct shash_desc *desc, u8 *dst) put_unaligned_be64(ctx->digest[1], dst); put_unaligned_be64(ctx->digest[0], dst + 8); - *ctx = (struct ghash_desc_ctx){}; + memzero_explicit(ctx, sizeof(*ctx)); return 0; } diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index f33ada70c4ed..683de671741a 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -177,7 +177,7 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } poly1305_emit(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; + memzero_explicit(dctx, sizeof(*dctx)); } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c index 9a4bbfc45f40..e5a2936f0886 100644 --- a/arch/arm64/crypto/sha3-ce-glue.c +++ b/arch/arm64/crypto/sha3-ce-glue.c @@ -94,7 +94,7 @@ static int sha3_final(struct shash_desc *desc, u8 *out) if (digest_size & 4) put_unaligned_le32(sctx->st[i], (__le32 *)digest); - *sctx = (struct sha3_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index e508dbd91813..64d09520d279 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -209,7 +209,7 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } poly1305_simd_emit(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; + memzero_explicit(dctx, sizeof(*dctx)); } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h index 20fd1f7468af..a5d6033efef7 100644 --- a/include/crypto/sha1_base.h +++ b/include/crypto/sha1_base.h @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -101,7 +102,7 @@ static inline int sha1_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) put_unaligned_be32(sctx->state[i], digest++); - *sctx = (struct sha1_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index 6ded110783ae..93f9fd21cc06 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -105,7 +106,7 @@ static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be32)) put_unaligned_be32(sctx->state[i], digest++); - *sctx = (struct sha256_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index fb19c77494dc..93ab73baa38e 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -126,7 +127,7 @@ static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be64)) put_unaligned_be64(sctx->state[i], digest++); - *sctx = (struct sha512_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h index 1cbf9aa1fe52..2f3a32ab97bb 100644 --- a/include/crypto/sm3_base.h +++ b/include/crypto/sm3_base.h @@ -13,6 +13,7 @@ #include #include #include +#include #include typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks); @@ -104,7 +105,7 @@ static inline int sm3_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; i < SM3_DIGEST_SIZE / sizeof(__be32); i++) put_unaligned_be32(sctx->state[i], digest++); - *sctx = (struct sm3_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } -- cgit v1.2.3 From 519a0d7e495a6d3ce62594e485aea2a3a4a2ca0a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 27 Oct 2020 00:00:27 +0100 Subject: crypto: arm64/poly1305-neon - reorder PAC authentication with SP update PAC pointer authentication signs the return address against the value of the stack pointer, to prevent stack overrun exploits from corrupting the control flow. However, this requires that the AUTIASP is issued with SP holding the same value as it held when the PAC value was generated. The Poly1305 NEON code got this wrong, resulting in crashes on PAC capable hardware. Fixes: f569ca164751 ("crypto: arm64/poly1305 - incorporate OpenSSL/CRYPTOGAMS ...") Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/poly1305-armv8.pl | 2 +- arch/arm64/crypto/poly1305-core.S_shipped | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/poly1305-armv8.pl b/arch/arm64/crypto/poly1305-armv8.pl index 6e5576d19af8..cbc980fb02e3 100644 --- a/arch/arm64/crypto/poly1305-armv8.pl +++ b/arch/arm64/crypto/poly1305-armv8.pl @@ -840,7 +840,6 @@ poly1305_blocks_neon: ldp d14,d15,[sp,#64] addp $ACC2,$ACC2,$ACC2 ldr x30,[sp,#8] - .inst 0xd50323bf // autiasp //////////////////////////////////////////////////////////////// // lazy reduction, but without narrowing @@ -882,6 +881,7 @@ poly1305_blocks_neon: str x4,[$ctx,#8] // set is_base2_26 ldr x29,[sp],#80 + .inst 0xd50323bf // autiasp ret .size poly1305_blocks_neon,.-poly1305_blocks_neon diff --git a/arch/arm64/crypto/poly1305-core.S_shipped b/arch/arm64/crypto/poly1305-core.S_shipped index 8d1c4e420ccd..fb2822abf63a 100644 --- a/arch/arm64/crypto/poly1305-core.S_shipped +++ b/arch/arm64/crypto/poly1305-core.S_shipped @@ -779,7 +779,6 @@ poly1305_blocks_neon: ldp d14,d15,[sp,#64] addp v21.2d,v21.2d,v21.2d ldr x30,[sp,#8] - .inst 0xd50323bf // autiasp //////////////////////////////////////////////////////////////// // lazy reduction, but without narrowing @@ -821,6 +820,7 @@ poly1305_blocks_neon: str x4,[x0,#8] // set is_base2_26 ldr x29,[sp],#80 + .inst 0xd50323bf // autiasp ret .size poly1305_blocks_neon,.-poly1305_blocks_neon -- cgit v1.2.3 From a2715fbdc6fc387e85211df917a4778761ec693d Mon Sep 17 00:00:00 2001 From: Horia Geantă Date: Wed, 28 Oct 2020 11:03:20 +0200 Subject: crypto: arm/aes-neonbs - fix usage of cbc(aes) fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loading the module deadlocks since: -local cbc(aes) implementation needs a fallback and -crypto API tries to find one but the request_module() resolves back to the same module Fix this by changing the module alias for cbc(aes) and using the NEED_FALLBACK flag when requesting for a fallback algorithm. Fixes: 00b99ad2bac2 ("crypto: arm/aes-neonbs - Use generic cbc encryption path") Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-neonbs-glue.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index bda8bf17631e..f70af1d0514b 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -19,7 +19,7 @@ MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("ecb(aes)"); -MODULE_ALIAS_CRYPTO("cbc(aes)"); +MODULE_ALIAS_CRYPTO("cbc(aes)-all"); MODULE_ALIAS_CRYPTO("ctr(aes)"); MODULE_ALIAS_CRYPTO("xts(aes)"); @@ -191,7 +191,8 @@ static int cbc_init(struct crypto_skcipher *tfm) struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); unsigned int reqsize; - ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC); + ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->enc_tfm)) return PTR_ERR(ctx->enc_tfm); @@ -441,7 +442,8 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, + .base.cra_flags = CRYPTO_ALG_INTERNAL | + CRYPTO_ALG_NEED_FALLBACK, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, -- cgit v1.2.3 From 86cd97ec4b943af35562a74688bc4e909b32c3d1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Nov 2020 17:28:09 +0100 Subject: crypto: arm/chacha-neon - optimize for non-block size multiples The current NEON based ChaCha implementation for ARM is optimized for multiples of 4x the ChaCha block size (64 bytes). This makes sense for block encryption, but given that ChaCha is also often used in the context of networking, it makes sense to consider arbitrary length inputs as well. For example, WireGuard typically uses 1420 byte packets, and performing ChaCha encryption involves 5 invocations of chacha_4block_xor_neon() and 3 invocations of chacha_block_xor_neon(), where the last one also involves a memcpy() using a buffer on the stack to process the final chunk of 1420 % 64 == 12 bytes. Let's optimize for this case as well, by letting chacha_4block_xor_neon() deal with any input size between 64 and 256 bytes, using NEON permutation instructions and overlapping loads and stores. This way, the 140 byte tail of a 1420 byte input buffer can simply be processed in one go. This results in the following performance improvements for 1420 byte blocks, without significant impact on power-of-2 input sizes. (Note that Raspberry Pi is widely used in combination with a 32-bit kernel, even though the core is 64-bit capable) Cortex-A8 (BeagleBone) : 7% Cortex-A15 (Calxeda Midway) : 21% Cortex-A53 (Raspberry Pi 3) : 3% Cortex-A72 (Raspberry Pi 4) : 19% Cc: Eric Biggers Cc: "Jason A . Donenfeld" Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/chacha-glue.c | 34 ++++++------- arch/arm/crypto/chacha-neon-core.S | 97 +++++++++++++++++++++++++++++++++++--- 2 files changed, 107 insertions(+), 24 deletions(-) (limited to 'arch') diff --git a/arch/arm/crypto/chacha-glue.c b/arch/arm/crypto/chacha-glue.c index 59da6c0b63b6..7b5cf8430c6d 100644 --- a/arch/arm/crypto/chacha-glue.c +++ b/arch/arm/crypto/chacha-glue.c @@ -23,7 +23,7 @@ asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src, int nrounds); asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src, - int nrounds); + int nrounds, unsigned int nbytes); asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds); asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds); @@ -42,24 +42,24 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, { u8 buf[CHACHA_BLOCK_SIZE]; - while (bytes >= CHACHA_BLOCK_SIZE * 4) { - chacha_4block_xor_neon(state, dst, src, nrounds); - bytes -= CHACHA_BLOCK_SIZE * 4; - src += CHACHA_BLOCK_SIZE * 4; - dst += CHACHA_BLOCK_SIZE * 4; - state[12] += 4; - } - while (bytes >= CHACHA_BLOCK_SIZE) { - chacha_block_xor_neon(state, dst, src, nrounds); - bytes -= CHACHA_BLOCK_SIZE; - src += CHACHA_BLOCK_SIZE; - dst += CHACHA_BLOCK_SIZE; - state[12]++; + while (bytes > CHACHA_BLOCK_SIZE) { + unsigned int l = min(bytes, CHACHA_BLOCK_SIZE * 4U); + + chacha_4block_xor_neon(state, dst, src, nrounds, l); + bytes -= l; + src += l; + dst += l; + state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } if (bytes) { - memcpy(buf, src, bytes); - chacha_block_xor_neon(state, buf, buf, nrounds); - memcpy(dst, buf, bytes); + const u8 *s = src; + u8 *d = dst; + + if (bytes != CHACHA_BLOCK_SIZE) + s = d = memcpy(buf, src, bytes); + chacha_block_xor_neon(state, d, s, nrounds); + if (d != dst) + memcpy(dst, buf, bytes); } } diff --git a/arch/arm/crypto/chacha-neon-core.S b/arch/arm/crypto/chacha-neon-core.S index eb22926d4912..13d12f672656 100644 --- a/arch/arm/crypto/chacha-neon-core.S +++ b/arch/arm/crypto/chacha-neon-core.S @@ -47,6 +47,7 @@ */ #include +#include .text .fpu neon @@ -205,7 +206,7 @@ ENDPROC(hchacha_block_neon) .align 5 ENTRY(chacha_4block_xor_neon) - push {r4-r5} + push {r4, lr} mov r4, sp // preserve the stack pointer sub ip, sp, #0x20 // allocate a 32 byte buffer bic ip, ip, #0x1f // aligned to 32 bytes @@ -229,10 +230,10 @@ ENTRY(chacha_4block_xor_neon) vld1.32 {q0-q1}, [r0] vld1.32 {q2-q3}, [ip] - adr r5, .Lctrinc + adr lr, .Lctrinc vdup.32 q15, d7[1] vdup.32 q14, d7[0] - vld1.32 {q4}, [r5, :128] + vld1.32 {q4}, [lr, :128] vdup.32 q13, d6[1] vdup.32 q12, d6[0] vdup.32 q11, d5[1] @@ -455,7 +456,7 @@ ENTRY(chacha_4block_xor_neon) // Re-interleave the words in the first two rows of each block (x0..7). // Also add the counter values 0-3 to x12[0-3]. - vld1.32 {q8}, [r5, :128] // load counter values 0-3 + vld1.32 {q8}, [lr, :128] // load counter values 0-3 vzip.32 q0, q1 // => (0 1 0 1) (0 1 0 1) vzip.32 q2, q3 // => (2 3 2 3) (2 3 2 3) vzip.32 q4, q5 // => (4 5 4 5) (4 5 4 5) @@ -493,6 +494,8 @@ ENTRY(chacha_4block_xor_neon) // Re-interleave the words in the last two rows of each block (x8..15). vld1.32 {q8-q9}, [sp, :256] + mov sp, r4 // restore original stack pointer + ldr r4, [r4, #8] // load number of bytes vzip.32 q12, q13 // => (12 13 12 13) (12 13 12 13) vzip.32 q14, q15 // => (14 15 14 15) (14 15 14 15) vzip.32 q8, q9 // => (8 9 8 9) (8 9 8 9) @@ -520,41 +523,121 @@ ENTRY(chacha_4block_xor_neon) // XOR the rest of the data with the keystream vld1.8 {q0-q1}, [r2]! + subs r4, r4, #96 veor q0, q0, q8 veor q1, q1, q12 + ble .Lle96 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q2 veor q1, q1, q6 + ble .Lle128 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q10 veor q1, q1, q14 + ble .Lle160 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q4 veor q1, q1, q5 + ble .Lle192 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q9 veor q1, q1, q13 + ble .Lle224 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q3 veor q1, q1, q7 + blt .Llt256 +.Lout: vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2] - mov sp, r4 // restore original stack pointer veor q0, q0, q11 veor q1, q1, q15 vst1.8 {q0-q1}, [r1] - pop {r4-r5} - bx lr + pop {r4, pc} + +.Lle192: + vmov q4, q9 + vmov q5, q13 + +.Lle160: + // nothing to do + +.Lfinalblock: + // Process the final block if processing less than 4 full blocks. + // Entered with 32 bytes of ChaCha cipher stream in q4-q5, and the + // previous 32 byte output block that still needs to be written at + // [r1] in q0-q1. + beq .Lfullblock + +.Lpartialblock: + adr lr, .Lpermute + 32 + add r2, r2, r4 + add lr, lr, r4 + add r4, r4, r1 + + vld1.8 {q2-q3}, [lr] + vld1.8 {q6-q7}, [r2] + + add r4, r4, #32 + + vtbl.8 d4, {q4-q5}, d4 + vtbl.8 d5, {q4-q5}, d5 + vtbl.8 d6, {q4-q5}, d6 + vtbl.8 d7, {q4-q5}, d7 + + veor q6, q6, q2 + veor q7, q7, q3 + + vst1.8 {q6-q7}, [r4] // overlapping stores + vst1.8 {q0-q1}, [r1] + pop {r4, pc} + +.Lfullblock: + vmov q11, q4 + vmov q15, q5 + b .Lout +.Lle96: + vmov q4, q2 + vmov q5, q6 + b .Lfinalblock +.Lle128: + vmov q4, q10 + vmov q5, q14 + b .Lfinalblock +.Lle224: + vmov q4, q3 + vmov q5, q7 + b .Lfinalblock +.Llt256: + vmov q4, q11 + vmov q5, q15 + b .Lpartialblock ENDPROC(chacha_4block_xor_neon) + + .align L1_CACHE_SHIFT +.Lpermute: + .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + .byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 + .byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f + .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + .byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 + .byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f -- cgit v1.2.3 From c4fc6328d6c67690a7e6e03f43a5a976a13120ef Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 6 Nov 2020 17:39:38 +0100 Subject: crypto: arm64/chacha - simplify tail block handling Based on lessons learnt from optimizing the 32-bit version of this driver, we can simplify the arm64 version considerably, by reordering the final two stores when the last block is not a multiple of 64 bytes. This removes the need to use permutation instructions to calculate the elements that are clobbered by the final overlapping store, given that the store of the penultimate block now follows it, and that one carries the correct values for those elements already. While at it, simplify the overlapping loads as well, by calculating the address of the final overlapping load upfront, and switching to this address for every load that would otherwise extend past the end of the source buffer. There is no impact on performance, but the resulting code is substantially smaller and easier to follow. Cc: Eric Biggers Cc: "Jason A . Donenfeld" Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/chacha-neon-core.S | 193 +++++++++++++---------------------- 1 file changed, 69 insertions(+), 124 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/chacha-neon-core.S b/arch/arm64/crypto/chacha-neon-core.S index e90386a7db8e..b70ac76f2610 100644 --- a/arch/arm64/crypto/chacha-neon-core.S +++ b/arch/arm64/crypto/chacha-neon-core.S @@ -195,7 +195,6 @@ SYM_FUNC_START(chacha_4block_xor_neon) adr_l x10, .Lpermute and x5, x4, #63 add x10, x10, x5 - add x11, x10, #64 // // This function encrypts four consecutive ChaCha blocks by loading @@ -645,11 +644,11 @@ CPU_BE( rev a15, a15 ) zip2 v31.4s, v14.4s, v15.4s eor a15, a15, w9 - mov x3, #64 + add x3, x2, x4 + sub x3, x3, #128 // start of last block + subs x5, x4, #128 - add x6, x5, x2 - csel x3, x3, xzr, ge - csel x2, x2, x6, ge + csel x2, x2, x3, ge // interleave 64-bit words in state n, n+2 zip1 v0.2d, v16.2d, v18.2d @@ -658,13 +657,10 @@ CPU_BE( rev a15, a15 ) zip1 v8.2d, v17.2d, v19.2d zip2 v12.2d, v17.2d, v19.2d stp a2, a3, [x1, #-56] - ld1 {v16.16b-v19.16b}, [x2], x3 subs x6, x4, #192 - ccmp x3, xzr, #4, lt - add x7, x6, x2 - csel x3, x3, xzr, eq - csel x2, x2, x7, eq + ld1 {v16.16b-v19.16b}, [x2], #64 + csel x2, x2, x3, ge zip1 v1.2d, v20.2d, v22.2d zip2 v5.2d, v20.2d, v22.2d @@ -672,13 +668,10 @@ CPU_BE( rev a15, a15 ) zip1 v9.2d, v21.2d, v23.2d zip2 v13.2d, v21.2d, v23.2d stp a6, a7, [x1, #-40] - ld1 {v20.16b-v23.16b}, [x2], x3 subs x7, x4, #256 - ccmp x3, xzr, #4, lt - add x8, x7, x2 - csel x3, x3, xzr, eq - csel x2, x2, x8, eq + ld1 {v20.16b-v23.16b}, [x2], #64 + csel x2, x2, x3, ge zip1 v2.2d, v24.2d, v26.2d zip2 v6.2d, v24.2d, v26.2d @@ -686,12 +679,10 @@ CPU_BE( rev a15, a15 ) zip1 v10.2d, v25.2d, v27.2d zip2 v14.2d, v25.2d, v27.2d stp a10, a11, [x1, #-24] - ld1 {v24.16b-v27.16b}, [x2], x3 subs x8, x4, #320 - ccmp x3, xzr, #4, lt - add x9, x8, x2 - csel x2, x2, x9, eq + ld1 {v24.16b-v27.16b}, [x2], #64 + csel x2, x2, x3, ge zip1 v3.2d, v28.2d, v30.2d zip2 v7.2d, v28.2d, v30.2d @@ -699,151 +690,105 @@ CPU_BE( rev a15, a15 ) zip1 v11.2d, v29.2d, v31.2d zip2 v15.2d, v29.2d, v31.2d stp a14, a15, [x1, #-8] + + tbnz x5, #63, .Lt128 ld1 {v28.16b-v31.16b}, [x2] // xor with corresponding input, write to output - tbnz x5, #63, 0f eor v16.16b, v16.16b, v0.16b eor v17.16b, v17.16b, v1.16b eor v18.16b, v18.16b, v2.16b eor v19.16b, v19.16b, v3.16b - st1 {v16.16b-v19.16b}, [x1], #64 - cbz x5, .Lout - tbnz x6, #63, 1f + tbnz x6, #63, .Lt192 + eor v20.16b, v20.16b, v4.16b eor v21.16b, v21.16b, v5.16b eor v22.16b, v22.16b, v6.16b eor v23.16b, v23.16b, v7.16b - st1 {v20.16b-v23.16b}, [x1], #64 - cbz x6, .Lout - tbnz x7, #63, 2f + st1 {v16.16b-v19.16b}, [x1], #64 + tbnz x7, #63, .Lt256 + eor v24.16b, v24.16b, v8.16b eor v25.16b, v25.16b, v9.16b eor v26.16b, v26.16b, v10.16b eor v27.16b, v27.16b, v11.16b - st1 {v24.16b-v27.16b}, [x1], #64 - cbz x7, .Lout - tbnz x8, #63, 3f + st1 {v20.16b-v23.16b}, [x1], #64 + tbnz x8, #63, .Lt320 + eor v28.16b, v28.16b, v12.16b eor v29.16b, v29.16b, v13.16b eor v30.16b, v30.16b, v14.16b eor v31.16b, v31.16b, v15.16b + + st1 {v24.16b-v27.16b}, [x1], #64 st1 {v28.16b-v31.16b}, [x1] .Lout: frame_pop ret - // fewer than 128 bytes of in/output -0: ld1 {v8.16b}, [x10] - ld1 {v9.16b}, [x11] - movi v10.16b, #16 - sub x2, x1, #64 - add x1, x1, x5 - ld1 {v16.16b-v19.16b}, [x2] - tbl v4.16b, {v0.16b-v3.16b}, v8.16b - tbx v20.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v5.16b, {v0.16b-v3.16b}, v8.16b - tbx v21.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v6.16b, {v0.16b-v3.16b}, v8.16b - tbx v22.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v7.16b, {v0.16b-v3.16b}, v8.16b - tbx v23.16b, {v16.16b-v19.16b}, v9.16b - - eor v20.16b, v20.16b, v4.16b - eor v21.16b, v21.16b, v5.16b - eor v22.16b, v22.16b, v6.16b - eor v23.16b, v23.16b, v7.16b - st1 {v20.16b-v23.16b}, [x1] - b .Lout - // fewer than 192 bytes of in/output -1: ld1 {v8.16b}, [x10] - ld1 {v9.16b}, [x11] - movi v10.16b, #16 - add x1, x1, x6 - tbl v0.16b, {v4.16b-v7.16b}, v8.16b - tbx v20.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v1.16b, {v4.16b-v7.16b}, v8.16b - tbx v21.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v2.16b, {v4.16b-v7.16b}, v8.16b - tbx v22.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v3.16b, {v4.16b-v7.16b}, v8.16b - tbx v23.16b, {v16.16b-v19.16b}, v9.16b - - eor v20.16b, v20.16b, v0.16b - eor v21.16b, v21.16b, v1.16b - eor v22.16b, v22.16b, v2.16b - eor v23.16b, v23.16b, v3.16b - st1 {v20.16b-v23.16b}, [x1] +.Lt192: cbz x5, 1f // exactly 128 bytes? + ld1 {v28.16b-v31.16b}, [x10] + add x5, x5, x1 + tbl v28.16b, {v4.16b-v7.16b}, v28.16b + tbl v29.16b, {v4.16b-v7.16b}, v29.16b + tbl v30.16b, {v4.16b-v7.16b}, v30.16b + tbl v31.16b, {v4.16b-v7.16b}, v31.16b + +0: eor v20.16b, v20.16b, v28.16b + eor v21.16b, v21.16b, v29.16b + eor v22.16b, v22.16b, v30.16b + eor v23.16b, v23.16b, v31.16b + st1 {v20.16b-v23.16b}, [x5] // overlapping stores +1: st1 {v16.16b-v19.16b}, [x1] b .Lout + // fewer than 128 bytes of in/output +.Lt128: ld1 {v28.16b-v31.16b}, [x10] + add x5, x5, x1 + sub x1, x1, #64 + tbl v28.16b, {v0.16b-v3.16b}, v28.16b + tbl v29.16b, {v0.16b-v3.16b}, v29.16b + tbl v30.16b, {v0.16b-v3.16b}, v30.16b + tbl v31.16b, {v0.16b-v3.16b}, v31.16b + ld1 {v16.16b-v19.16b}, [x1] // reload first output block + b 0b + // fewer than 256 bytes of in/output -2: ld1 {v4.16b}, [x10] - ld1 {v5.16b}, [x11] - movi v6.16b, #16 - add x1, x1, x7 +.Lt256: cbz x6, 2f // exactly 192 bytes? + ld1 {v4.16b-v7.16b}, [x10] + add x6, x6, x1 tbl v0.16b, {v8.16b-v11.16b}, v4.16b - tbx v24.16b, {v20.16b-v23.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v1.16b, {v8.16b-v11.16b}, v4.16b - tbx v25.16b, {v20.16b-v23.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v2.16b, {v8.16b-v11.16b}, v4.16b - tbx v26.16b, {v20.16b-v23.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v3.16b, {v8.16b-v11.16b}, v4.16b - tbx v27.16b, {v20.16b-v23.16b}, v5.16b - - eor v24.16b, v24.16b, v0.16b - eor v25.16b, v25.16b, v1.16b - eor v26.16b, v26.16b, v2.16b - eor v27.16b, v27.16b, v3.16b - st1 {v24.16b-v27.16b}, [x1] + tbl v1.16b, {v8.16b-v11.16b}, v5.16b + tbl v2.16b, {v8.16b-v11.16b}, v6.16b + tbl v3.16b, {v8.16b-v11.16b}, v7.16b + + eor v28.16b, v28.16b, v0.16b + eor v29.16b, v29.16b, v1.16b + eor v30.16b, v30.16b, v2.16b + eor v31.16b, v31.16b, v3.16b + st1 {v28.16b-v31.16b}, [x6] // overlapping stores +2: st1 {v20.16b-v23.16b}, [x1] b .Lout // fewer than 320 bytes of in/output -3: ld1 {v4.16b}, [x10] - ld1 {v5.16b}, [x11] - movi v6.16b, #16 - add x1, x1, x8 +.Lt320: cbz x7, 3f // exactly 256 bytes? + ld1 {v4.16b-v7.16b}, [x10] + add x7, x7, x1 tbl v0.16b, {v12.16b-v15.16b}, v4.16b - tbx v28.16b, {v24.16b-v27.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v1.16b, {v12.16b-v15.16b}, v4.16b - tbx v29.16b, {v24.16b-v27.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v2.16b, {v12.16b-v15.16b}, v4.16b - tbx v30.16b, {v24.16b-v27.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v3.16b, {v12.16b-v15.16b}, v4.16b - tbx v31.16b, {v24.16b-v27.16b}, v5.16b + tbl v1.16b, {v12.16b-v15.16b}, v5.16b + tbl v2.16b, {v12.16b-v15.16b}, v6.16b + tbl v3.16b, {v12.16b-v15.16b}, v7.16b eor v28.16b, v28.16b, v0.16b eor v29.16b, v29.16b, v1.16b eor v30.16b, v30.16b, v2.16b eor v31.16b, v31.16b, v3.16b - st1 {v28.16b-v31.16b}, [x1] + st1 {v28.16b-v31.16b}, [x7] // overlapping stores +3: st1 {v24.16b-v27.16b}, [x1] b .Lout SYM_FUNC_END(chacha_4block_xor_neon) @@ -851,7 +796,7 @@ SYM_FUNC_END(chacha_4block_xor_neon) .align L1_CACHE_SHIFT .Lpermute: .set .Li, 0 - .rept 192 + .rept 128 .byte (.Li - 64) .set .Li, .Li + 1 .endr -- cgit v1.2.3 From 3ad99c22cebee84034bc53e42f5115dbc97490e8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 10 Nov 2020 10:10:42 +0100 Subject: crypto: arm64/gcm - move authentication tag check to SIMD domain Instead of copying the calculated authentication tag to memory and calling crypto_memneq() to verify it, use vector bytewise compare and min across vector instructions to decide whether the tag is valid. This is more efficient, and given that the tag is only transiently held in a NEON register, it is also safer, given that calculated tags for failed decryptions should be withheld. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/ghash-ce-core.S | 15 +++++++++++++ arch/arm64/crypto/ghash-ce-glue.c | 46 ++++++++++++++++++++++++--------------- 2 files changed, 43 insertions(+), 18 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index 6b958dcdf136..7868330dd54e 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -544,7 +544,22 @@ CPU_LE( rev w8, w8 ) ext XL.16b, XL.16b, XL.16b, #8 rev64 XL.16b, XL.16b eor XL.16b, XL.16b, KS0.16b + + .if \enc == 1 st1 {XL.16b}, [x10] // store tag + .else + ldp x11, x12, [sp, #40] // load tag pointer and authsize + adr_l x17, .Lpermute_table + ld1 {KS0.16b}, [x11] // load supplied tag + add x17, x17, x12 + ld1 {KS1.16b}, [x17] // load permute vector + + cmeq XL.16b, XL.16b, KS0.16b // compare tags + mvn XL.16b, XL.16b // -1 for fail, 0 for pass + tbl XL.16b, {XL.16b}, KS1.16b // keep authsize bytes only + sminv b0, XL.16b // signed minimum across XL + smov w0, v0.b[0] // return b0 + .endif 4: ldp x29, x30, [sp], #32 ret diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 2427e2f3a9a1..720cd3a58da3 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -55,10 +55,10 @@ asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[], u64 const h[][2], u64 dg[], u8 ctr[], u32 const rk[], int rounds, u8 tag[]); - -asmlinkage void pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], - u64 const h[][2], u64 dg[], u8 ctr[], - u32 const rk[], int rounds, u8 tag[]); +asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], + u64 const h[][2], u64 dg[], u8 ctr[], + u32 const rk[], int rounds, const u8 l[], + const u8 tag[], u64 authsize); static int ghash_init(struct shash_desc *desc) { @@ -458,6 +458,7 @@ static int gcm_decrypt(struct aead_request *req) unsigned int authsize = crypto_aead_authsize(aead); int nrounds = num_rounds(&ctx->aes_key); struct skcipher_walk walk; + u8 otag[AES_BLOCK_SIZE]; u8 buf[AES_BLOCK_SIZE]; u8 iv[AES_BLOCK_SIZE]; u64 dg[2] = {}; @@ -474,9 +475,15 @@ static int gcm_decrypt(struct aead_request *req) memcpy(iv, req->iv, GCM_IV_SIZE); put_unaligned_be32(2, iv + GCM_IV_SIZE); + scatterwalk_map_and_copy(otag, req->src, + req->assoclen + req->cryptlen - authsize, + authsize, 0); + err = skcipher_walk_aead_decrypt(&walk, req, false); if (likely(crypto_simd_usable())) { + int ret; + do { const u8 *src = walk.src.virt.addr; u8 *dst = walk.dst.virt.addr; @@ -493,9 +500,10 @@ static int gcm_decrypt(struct aead_request *req) } kernel_neon_begin(); - pmull_gcm_decrypt(nbytes, dst, src, ctx->ghash_key.h, - dg, iv, ctx->aes_key.key_enc, nrounds, - tag); + ret = pmull_gcm_decrypt(nbytes, dst, src, + ctx->ghash_key.h, + dg, iv, ctx->aes_key.key_enc, + nrounds, tag, otag, authsize); kernel_neon_end(); if (unlikely(!nbytes)) @@ -507,6 +515,11 @@ static int gcm_decrypt(struct aead_request *req) err = skcipher_walk_done(&walk, walk.nbytes - nbytes); } while (walk.nbytes); + + if (err) + return err; + if (ret) + return -EBADMSG; } else { while (walk.nbytes >= AES_BLOCK_SIZE) { int blocks = walk.nbytes / AES_BLOCK_SIZE; @@ -548,23 +561,20 @@ static int gcm_decrypt(struct aead_request *req) err = skcipher_walk_done(&walk, 0); } + if (err) + return err; + put_unaligned_be64(dg[1], tag); put_unaligned_be64(dg[0], tag + 8); put_unaligned_be32(1, iv + GCM_IV_SIZE); aes_encrypt(&ctx->aes_key, iv, iv); crypto_xor(tag, iv, AES_BLOCK_SIZE); - } - - if (err) - return err; - /* compare calculated auth tag with the stored one */ - scatterwalk_map_and_copy(buf, req->src, - req->assoclen + req->cryptlen - authsize, - authsize, 0); - - if (crypto_memneq(tag, buf, authsize)) - return -EBADMSG; + if (crypto_memneq(tag, otag, authsize)) { + memzero_explicit(tag, AES_BLOCK_SIZE); + return -EBADMSG; + } + } return 0; } -- cgit v1.2.3 From a24d22b225ce158651378869a6b88105c4bdb887 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 12 Nov 2020 21:20:21 -0800 Subject: crypto: sha - split sha.h into sha1.h and sha2.h Currently contains declarations for both SHA-1 and SHA-2, and contains declarations for SHA-3. This organization is inconsistent, but more importantly SHA-1 is no longer considered to be cryptographically secure. So to the extent possible, SHA-1 shouldn't be grouped together with any of the other SHA versions, and usage of it should be phased out. Therefore, split into two headers and , and make everyone explicitly specify whether they want the declarations for SHA-1, SHA-2, or both. This avoids making the SHA-1 declarations visible to files that don't want anything to do with SHA-1. It also prepares for potentially moving sha1.h into a new insecure/ or dangerous/ directory. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Acked-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- arch/arm/crypto/sha1-ce-glue.c | 2 +- arch/arm/crypto/sha1.h | 2 +- arch/arm/crypto/sha1_glue.c | 2 +- arch/arm/crypto/sha1_neon_glue.c | 2 +- arch/arm/crypto/sha2-ce-glue.c | 2 +- arch/arm/crypto/sha256_glue.c | 2 +- arch/arm/crypto/sha256_neon_glue.c | 2 +- arch/arm/crypto/sha512-glue.c | 2 +- arch/arm/crypto/sha512-neon-glue.c | 2 +- arch/arm64/crypto/aes-glue.c | 2 +- arch/arm64/crypto/sha1-ce-glue.c | 2 +- arch/arm64/crypto/sha2-ce-glue.c | 2 +- arch/arm64/crypto/sha256-glue.c | 2 +- arch/arm64/crypto/sha512-ce-glue.c | 2 +- arch/arm64/crypto/sha512-glue.c | 2 +- arch/mips/cavium-octeon/crypto/octeon-sha1.c | 2 +- arch/mips/cavium-octeon/crypto/octeon-sha256.c | 2 +- arch/mips/cavium-octeon/crypto/octeon-sha512.c | 2 +- arch/powerpc/crypto/sha1-spe-glue.c | 2 +- arch/powerpc/crypto/sha1.c | 2 +- arch/powerpc/crypto/sha256-spe-glue.c | 2 +- arch/s390/crypto/sha.h | 3 +- arch/s390/crypto/sha1_s390.c | 2 +- arch/s390/crypto/sha256_s390.c | 2 +- arch/s390/crypto/sha3_256_s390.c | 1 - arch/s390/crypto/sha3_512_s390.c | 1 - arch/s390/crypto/sha512_s390.c | 2 +- arch/s390/purgatory/purgatory.c | 2 +- arch/sparc/crypto/sha1_glue.c | 2 +- arch/sparc/crypto/sha256_glue.c | 2 +- arch/sparc/crypto/sha512_glue.c | 2 +- arch/x86/crypto/sha1_ssse3_glue.c | 2 +- arch/x86/crypto/sha256_ssse3_glue.c | 2 +- arch/x86/crypto/sha512_ssse3_glue.c | 2 +- arch/x86/purgatory/purgatory.c | 2 +- crypto/asymmetric_keys/asym_tpm.c | 2 +- crypto/sha1_generic.c | 2 +- crypto/sha256_generic.c | 2 +- crypto/sha512_generic.c | 2 +- drivers/char/random.c | 2 +- drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h | 2 +- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 3 +- drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h | 3 +- drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c | 3 +- drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h | 3 +- drivers/crypto/amcc/crypto4xx_alg.c | 2 +- drivers/crypto/amcc/crypto4xx_core.c | 2 +- drivers/crypto/atmel-authenc.h | 3 +- drivers/crypto/atmel-sha.c | 3 +- drivers/crypto/axis/artpec6_crypto.c | 3 +- drivers/crypto/bcm/cipher.c | 3 +- drivers/crypto/bcm/cipher.h | 3 +- drivers/crypto/bcm/spu.h | 3 +- drivers/crypto/caam/compat.h | 3 +- drivers/crypto/cavium/nitrox/nitrox_aead.c | 1 - drivers/crypto/ccp/ccp-crypto-sha.c | 3 +- drivers/crypto/ccp/ccp-crypto.h | 3 +- drivers/crypto/ccree/cc_driver.h | 3 +- drivers/crypto/chelsio/chcr_algo.c | 3 +- drivers/crypto/hisilicon/sec2/sec_crypto.c | 3 +- drivers/crypto/img-hash.c | 3 +- drivers/crypto/inside-secure/safexcel.h | 3 +- drivers/crypto/inside-secure/safexcel_cipher.c | 3 +- drivers/crypto/inside-secure/safexcel_hash.c | 3 +- drivers/crypto/ixp4xx_crypto.c | 2 +- drivers/crypto/marvell/cesa/hash.c | 3 +- drivers/crypto/marvell/octeontx/otx_cptvf_algs.c | 3 +- drivers/crypto/mediatek/mtk-sha.c | 3 +- drivers/crypto/mxs-dcp.c | 3 +- drivers/crypto/n2_core.c | 3 +- drivers/crypto/nx/nx-sha256.c | 2 +- drivers/crypto/nx/nx-sha512.c | 2 +- drivers/crypto/nx/nx.c | 2 +- drivers/crypto/omap-sham.c | 3 +- drivers/crypto/padlock-sha.c | 3 +- drivers/crypto/picoxcell_crypto.c | 3 +- drivers/crypto/qat/qat_common/qat_algs.c | 3 +- drivers/crypto/qce/common.c | 3 +- drivers/crypto/qce/core.c | 1 - drivers/crypto/qce/sha.h | 3 +- drivers/crypto/rockchip/rk3288_crypto.h | 3 +- drivers/crypto/s5p-sss.c | 3 +- drivers/crypto/sa2ul.c | 3 +- drivers/crypto/sa2ul.h | 3 +- drivers/crypto/sahara.c | 3 +- drivers/crypto/stm32/stm32-hash.c | 3 +- drivers/crypto/talitos.c | 3 +- drivers/crypto/ux500/hash/hash_core.c | 3 +- drivers/firmware/efi/embedded-firmware.c | 2 +- .../chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c | 3 +- .../ethernet/chelsio/inline_crypto/chtls/chtls.h | 3 +- drivers/nfc/s3fwrn5/firmware.c | 2 +- drivers/tee/tee_core.c | 2 +- fs/crypto/fname.c | 2 +- fs/crypto/hkdf.c | 2 +- fs/ubifs/auth.c | 1 - fs/verity/fsverity_private.h | 2 +- include/crypto/hash_info.h | 3 +- include/crypto/sha.h | 167 --------------------- include/crypto/sha1.h | 46 ++++++ include/crypto/sha1_base.h | 2 +- include/crypto/sha2.h | 134 +++++++++++++++++ include/crypto/sha256_base.h | 2 +- include/crypto/sha512_base.h | 2 +- include/linux/ccp.h | 3 +- include/linux/filter.h | 2 +- include/linux/purgatory.h | 2 +- kernel/crash_core.c | 2 +- kernel/kexec_core.c | 1 - kernel/kexec_file.c | 2 +- lib/crypto/sha256.c | 2 +- lib/digsig.c | 2 +- lib/sha1.c | 2 +- net/ipv6/seg6_hmac.c | 1 - net/mptcp/crypto.c | 2 +- net/mptcp/options.c | 2 +- net/mptcp/subflow.c | 2 +- security/integrity/integrity.h | 2 +- security/keys/encrypted-keys/encrypted.c | 2 +- security/keys/trusted-keys/trusted_tpm1.c | 2 +- sound/soc/codecs/cros_ec_codec.c | 2 +- 121 files changed, 335 insertions(+), 285 deletions(-) delete mode 100644 include/crypto/sha.h create mode 100644 include/crypto/sha1.h create mode 100644 include/crypto/sha2.h (limited to 'arch') diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c index e79b1fb4b4dc..de9100c67b37 100644 --- a/arch/arm/crypto/sha1-ce-glue.c +++ b/arch/arm/crypto/sha1-ce-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha1.h b/arch/arm/crypto/sha1.h index 758db3e9ff0a..b1b7e21da2c3 100644 --- a/arch/arm/crypto/sha1.h +++ b/arch/arm/crypto/sha1.h @@ -3,7 +3,7 @@ #define ASM_ARM_CRYPTO_SHA1_H #include -#include +#include extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, unsigned int len); diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index 4e954b3f7ecd..6c2b849e459d 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index 0071e5e4411a..cfe36ae0f3f5 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c index 87f0b62386c6..c62ce89dd3e0 100644 --- a/arch/arm/crypto/sha2-ce-glue.c +++ b/arch/arm/crypto/sha2-ce-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha256_glue.c b/arch/arm/crypto/sha256_glue.c index b8a4f79020cf..433ee4ddce6c 100644 --- a/arch/arm/crypto/sha256_glue.c +++ b/arch/arm/crypto/sha256_glue.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c index 79820b9e2541..701706262ef3 100644 --- a/arch/arm/crypto/sha256_neon_glue.c +++ b/arch/arm/crypto/sha256_neon_glue.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c index 8775aa42bbbe..0635a65aa488 100644 --- a/arch/arm/crypto/sha512-glue.c +++ b/arch/arm/crypto/sha512-glue.c @@ -6,7 +6,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-neon-glue.c index 96cb94403540..c879ad32db51 100644 --- a/arch/arm/crypto/sha512-neon-glue.c +++ b/arch/arm/crypto/sha512-neon-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 395bbf64b2ab..34b8a89197be 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index c63b99211db3..c93121bcfdeb 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 5e956d7582a5..31ba3da5e61b 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c index 77bc6e72abae..9462f6088b3f 100644 --- a/arch/arm64/crypto/sha256-glue.c +++ b/arch/arm64/crypto/sha256-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c index dc890a719f54..faa83f6cf376 100644 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c index 370ccb29602f..2acff1c7df5d 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c index 75e79b47abfe..30f1d75208a5 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index a682ce76716a..36cb92895d72 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -15,7 +15,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c index 50722a0cfb53..359f039820d8 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c index cb57be4ada61..b1e577cbf00c 100644 --- a/arch/powerpc/crypto/sha1-spe-glue.c +++ b/arch/powerpc/crypto/sha1-spe-glue.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c index b40dc50a6908..7a55d790cdb1 100644 --- a/arch/powerpc/crypto/sha1.c +++ b/arch/powerpc/crypto/sha1.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include void powerpc_sha_transform(u32 *state, const u8 *src); diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index ceb0b6c980b3..88530ae0791f 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h index ada2f98c27b7..65ea12fc87a1 100644 --- a/arch/s390/crypto/sha.h +++ b/arch/s390/crypto/sha.h @@ -11,7 +11,8 @@ #define _CRYPTO_ARCH_S390_SHA_H #include -#include +#include +#include #include /* must be big enough for the largest SHA variant */ diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index 698b1e6d3c14..a3fabf310a38 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include "sha.h" diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index b52c87e44939..24983f175676 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include "sha.h" diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c index 460cbbbaa44a..30ac49b635bf 100644 --- a/arch/s390/crypto/sha3_256_s390.c +++ b/arch/s390/crypto/sha3_256_s390.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c index 72cf460a53e5..e70d50f7620f 100644 --- a/arch/s390/crypto/sha3_512_s390.c +++ b/arch/s390/crypto/sha3_512_s390.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index ad29db085a18..29a6bd404c59 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -8,7 +8,7 @@ * Author(s): Jan Glauber (jang@de.ibm.com) */ #include -#include +#include #include #include #include diff --git a/arch/s390/purgatory/purgatory.c b/arch/s390/purgatory/purgatory.c index 0a423bcf6746..030efda05dbe 100644 --- a/arch/s390/purgatory/purgatory.c +++ b/arch/s390/purgatory/purgatory.c @@ -9,7 +9,7 @@ #include #include -#include +#include #include int verify_sha256_digest(void) diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c index dc017782be52..86a654cce5ab 100644 --- a/arch/sparc/crypto/sha1_glue.c +++ b/arch/sparc/crypto/sha1_glue.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c index ca2547df9652..60ec524cf9ca 100644 --- a/arch/sparc/crypto/sha256_glue.c +++ b/arch/sparc/crypto/sha256_glue.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c index 3b2ca732ff7a..273ce21918c1 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index 18200135603f..44340a1139e0 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index dd06249229e1..3a5f6be7dbba 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -35,7 +35,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index b0b05c93409e..30e70f4fe2f7 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/purgatory/purgatory.c b/arch/x86/purgatory/purgatory.c index 7b37a412f829..f03b64d9cb51 100644 --- a/arch/x86/purgatory/purgatory.c +++ b/arch/x86/purgatory/purgatory.c @@ -9,7 +9,7 @@ */ #include -#include +#include #include #include "../boot/string.h" diff --git a/crypto/asymmetric_keys/asym_tpm.c b/crypto/asymmetric_keys/asym_tpm.c index 378b18b9bc34..511932aa94a6 100644 --- a/crypto/asymmetric_keys/asym_tpm.c +++ b/crypto/asymmetric_keys/asym_tpm.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 1d43472fecbd..325b57fe28dc 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index 88156e3e2a33..3b377197236e 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index e34d09dd9971..c72d72ad828e 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/char/random.c b/drivers/char/random.c index 2a41b21623ae..5f3b8ac9d97b 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -336,7 +336,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h index 163962f9e284..5c291e4a6857 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index fa2f1b4fad7b..4927a6c82d32 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include "sun8i-ce.h" diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 558027516aed..cec781d5063c 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -16,7 +16,8 @@ #include #include #include -#include +#include +#include /* CE Registers */ #define CE_TDQ 0x00 diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index b6ab2054f217..11cbcbc83a7b 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include "sun8i-ss.h" diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h index 1a66457f4a20..28188685b910 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #define SS_START 1 diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 7729a637fb02..a3fa849b139a 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include "crypto4xx_reg_def.h" diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 2e3690f65786..8d1b918a0533 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/atmel-authenc.h b/drivers/crypto/atmel-authenc.h index c6530a1c8c20..45171e89a7d2 100644 --- a/drivers/crypto/atmel-authenc.h +++ b/drivers/crypto/atmel-authenc.h @@ -16,7 +16,8 @@ #include #include -#include +#include +#include #include "atmel-sha-regs.h" struct atmel_aes_dev; diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 0eb6f54e3b66..352d80cb5ae9 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -33,7 +33,8 @@ #include #include #include -#include +#include +#include #include #include #include "atmel-sha-regs.h" diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 809c3033ca74..9ad188cffd0d 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -28,7 +28,8 @@ #include #include #include -#include +#include +#include #include /* Max length of a line in all cache levels for Artpec SoCs. */ diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 50d169e61b41..30390a7324b2 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -26,11 +26,12 @@ #include #include #include -#include #include #include #include #include +#include +#include #include #include "util.h" diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index 035c8389cb3d..0ad5892b445d 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -16,7 +16,8 @@ #include #include #include -#include +#include +#include #include #include "spu.h" diff --git a/drivers/crypto/bcm/spu.h b/drivers/crypto/bcm/spu.h index dd132389bcaa..1c386a2d5506 100644 --- a/drivers/crypto/bcm/spu.h +++ b/drivers/crypto/bcm/spu.h @@ -17,7 +17,8 @@ #include #include -#include +#include +#include enum spu_cipher_alg { CIPHER_ALG_NONE = 0x0, diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index c3c22a8de4c0..c4f79764172b 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -34,7 +34,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c index e5d8607ecb1d..c93c4e41d267 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_aead.c +++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c index 8fbfdb9e8cd3..74fa5360e722 100644 --- a/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index aed3d2192d01..e42450d07168 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -19,7 +19,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index af77b2020350..ed2b2f13a256 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 13b908ea4873..f5a336634daa 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -53,7 +53,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 891e04914615..2eaa516b3231 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 91f555ccbb31..e813115d5432 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -19,7 +19,8 @@ #include #include -#include +#include +#include #define CR_RESET 0 #define CR_RESET_SET 1 diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 9045f2d7f4c6..ce1e611a163e 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -11,7 +11,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 9bcfb79a030f..d68ef16650d4 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -18,7 +18,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 56d5ccb5cc00..50fb6d90a2e0 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -8,7 +8,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 276012e7c482..8b0f17fc09fb 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index add7ea011c98..8cf9fd518d86 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -11,7 +11,8 @@ #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c index 90bb31329d4b..ccbef01888d4 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index 3d5d7d68b03b..f55aacdafbef 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -10,7 +10,8 @@ */ #include -#include +#include +#include #include "mtk-platform.h" #define SHA_ALIGN_MSK (sizeof(u32) - 1) diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index 909a7eb748e3..d6a7784d2988 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -17,7 +17,8 @@ #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 3642bf83d809..3b0bf6fea491 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c index 02fb53453195..90d9a37a57f6 100644 --- a/drivers/crypto/nx/nx-sha256.c +++ b/drivers/crypto/nx/nx-sha256.c @@ -8,7 +8,7 @@ */ #include -#include +#include #include #include #include diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c index 4c7a3e3eeebf..eb8627a0f317 100644 --- a/drivers/crypto/nx/nx-sha512.c +++ b/drivers/crypto/nx/nx-sha512.c @@ -8,7 +8,7 @@ */ #include -#include +#include #include #include diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 40882d6d52c1..0d2dc5be7f19 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index a3b38d2c92e7..ae0d320d3c60 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -35,7 +35,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index a697a4a3f2d0..6865c7f1fc1a 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -9,7 +9,8 @@ #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index fb34bf92861d..84f9c16d984c 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -8,7 +8,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 0fab8bb8ca59..b3a68d986417 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -6,7 +6,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c index 5006e74c40cd..a73db2a5637f 100644 --- a/drivers/crypto/qce/common.c +++ b/drivers/crypto/qce/common.c @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include "cipher.h" #include "common.h" diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index ea616b7259ae..5e6717f9bbda 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -13,7 +13,6 @@ #include #include #include -#include #include "core.h" #include "cipher.h" diff --git a/drivers/crypto/qce/sha.h b/drivers/crypto/qce/sha.h index d63526e3804d..a22695361f16 100644 --- a/drivers/crypto/qce/sha.h +++ b/drivers/crypto/qce/sha.h @@ -7,7 +7,8 @@ #define _SHA_H_ #include -#include +#include +#include #include "common.h" #include "core.h" diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h index 3db595570c9c..97278c2574ff 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.h +++ b/drivers/crypto/rockchip/rk3288_crypto.h @@ -12,7 +12,8 @@ #include #include -#include +#include +#include #define _SBF(v, f) ((v) << (f)) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 88a6c853ffd7..682c8a450a57 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -30,7 +30,8 @@ #include #include -#include +#include +#include #include #define _SBF(s, v) ((v) << (s)) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index c357010a159e..f300b0a5958a 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -25,7 +25,8 @@ #include #include #include -#include +#include +#include #include "sa2ul.h" diff --git a/drivers/crypto/sa2ul.h b/drivers/crypto/sa2ul.h index bb40df3876e5..f597ddecde34 100644 --- a/drivers/crypto/sa2ul.h +++ b/drivers/crypto/sa2ul.h @@ -13,7 +13,8 @@ #define _K3_SA2UL_ #include -#include +#include +#include #define SA_ENGINE_ENABLE_CONTROL 0x1000 diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index d60679c79822..8b5be29cb4dc 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index e3e25278a970..7ac0573ef663 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -25,7 +25,8 @@ #include #include #include -#include +#include +#include #include #define HASH_CR 0x00 diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index a713a35dc502..4fd85f31630a 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -31,7 +31,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index 3d407eebb2ba..da284b0ea1b2 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -31,7 +31,8 @@ #include #include -#include +#include +#include #include #include diff --git a/drivers/firmware/efi/embedded-firmware.c b/drivers/firmware/efi/embedded-firmware.c index 21ae0c48232a..f5be8e22305b 100644 --- a/drivers/firmware/efi/embedded-firmware.c +++ b/drivers/firmware/efi/embedded-firmware.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include /* Exported for use by lib/test_firmware.c only */ LIST_HEAD(efi_embedded_fw_list); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c index 072299b14b8d..47d9268a7e3c 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c @@ -51,7 +51,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h index 2d3dfdd2a716..65617752c630 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h @@ -9,7 +9,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/nfc/s3fwrn5/firmware.c b/drivers/nfc/s3fwrn5/firmware.c index ec930ee2c847..5d5ad8307211 100644 --- a/drivers/nfc/s3fwrn5/firmware.c +++ b/drivers/nfc/s3fwrn5/firmware.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include "s3fwrn5.h" #include "firmware.h" diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index f53bf336c0a2..d70d4be91096 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include "tee_private.h" #define TEE_NUM_DEVICES 32 diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 1fbe6c24d705..cf06ea3870eb 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include "fscrypt_private.h" diff --git a/fs/crypto/hkdf.c b/fs/crypto/hkdf.c index 0cba7928446d..e0ec21055505 100644 --- a/fs/crypto/hkdf.c +++ b/fs/crypto/hkdf.c @@ -10,7 +10,7 @@ */ #include -#include +#include #include "fscrypt_private.h" diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c index b93b3cd10bfd..0886d835f597 100644 --- a/fs/ubifs/auth.c +++ b/fs/ubifs/auth.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index e96d99d5145e..6a8f2e3cce6c 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -14,7 +14,7 @@ #define pr_fmt(fmt) "fs-verity: " fmt -#include +#include #include #include diff --git a/include/crypto/hash_info.h b/include/crypto/hash_info.h index eb9d2e368969..dd4f06785049 100644 --- a/include/crypto/hash_info.h +++ b/include/crypto/hash_info.h @@ -8,7 +8,8 @@ #ifndef _CRYPTO_HASH_INFO_H #define _CRYPTO_HASH_INFO_H -#include +#include +#include #include #include diff --git a/include/crypto/sha.h b/include/crypto/sha.h deleted file mode 100644 index 4ff3da816630..000000000000 --- a/include/crypto/sha.h +++ /dev/null @@ -1,167 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Common values for SHA algorithms - */ - -#ifndef _CRYPTO_SHA_H -#define _CRYPTO_SHA_H - -#include - -#define SHA1_DIGEST_SIZE 20 -#define SHA1_BLOCK_SIZE 64 - -#define SHA224_DIGEST_SIZE 28 -#define SHA224_BLOCK_SIZE 64 - -#define SHA256_DIGEST_SIZE 32 -#define SHA256_BLOCK_SIZE 64 - -#define SHA384_DIGEST_SIZE 48 -#define SHA384_BLOCK_SIZE 128 - -#define SHA512_DIGEST_SIZE 64 -#define SHA512_BLOCK_SIZE 128 - -#define SHA1_H0 0x67452301UL -#define SHA1_H1 0xefcdab89UL -#define SHA1_H2 0x98badcfeUL -#define SHA1_H3 0x10325476UL -#define SHA1_H4 0xc3d2e1f0UL - -#define SHA224_H0 0xc1059ed8UL -#define SHA224_H1 0x367cd507UL -#define SHA224_H2 0x3070dd17UL -#define SHA224_H3 0xf70e5939UL -#define SHA224_H4 0xffc00b31UL -#define SHA224_H5 0x68581511UL -#define SHA224_H6 0x64f98fa7UL -#define SHA224_H7 0xbefa4fa4UL - -#define SHA256_H0 0x6a09e667UL -#define SHA256_H1 0xbb67ae85UL -#define SHA256_H2 0x3c6ef372UL -#define SHA256_H3 0xa54ff53aUL -#define SHA256_H4 0x510e527fUL -#define SHA256_H5 0x9b05688cUL -#define SHA256_H6 0x1f83d9abUL -#define SHA256_H7 0x5be0cd19UL - -#define SHA384_H0 0xcbbb9d5dc1059ed8ULL -#define SHA384_H1 0x629a292a367cd507ULL -#define SHA384_H2 0x9159015a3070dd17ULL -#define SHA384_H3 0x152fecd8f70e5939ULL -#define SHA384_H4 0x67332667ffc00b31ULL -#define SHA384_H5 0x8eb44a8768581511ULL -#define SHA384_H6 0xdb0c2e0d64f98fa7ULL -#define SHA384_H7 0x47b5481dbefa4fa4ULL - -#define SHA512_H0 0x6a09e667f3bcc908ULL -#define SHA512_H1 0xbb67ae8584caa73bULL -#define SHA512_H2 0x3c6ef372fe94f82bULL -#define SHA512_H3 0xa54ff53a5f1d36f1ULL -#define SHA512_H4 0x510e527fade682d1ULL -#define SHA512_H5 0x9b05688c2b3e6c1fULL -#define SHA512_H6 0x1f83d9abfb41bd6bULL -#define SHA512_H7 0x5be0cd19137e2179ULL - -extern const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE]; - -extern const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE]; - -extern const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE]; - -extern const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE]; - -extern const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE]; - -struct sha1_state { - u32 state[SHA1_DIGEST_SIZE / 4]; - u64 count; - u8 buffer[SHA1_BLOCK_SIZE]; -}; - -struct sha256_state { - u32 state[SHA256_DIGEST_SIZE / 4]; - u64 count; - u8 buf[SHA256_BLOCK_SIZE]; -}; - -struct sha512_state { - u64 state[SHA512_DIGEST_SIZE / 8]; - u64 count[2]; - u8 buf[SHA512_BLOCK_SIZE]; -}; - -struct shash_desc; - -extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - -extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - -extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - -/* - * An implementation of SHA-1's compression function. Don't use in new code! - * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't - * the correct way to hash something with SHA-1 (use crypto_shash instead). - */ -#define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) -#define SHA1_WORKSPACE_WORDS 16 -void sha1_init(__u32 *buf); -void sha1_transform(__u32 *digest, const char *data, __u32 *W); - -/* - * Stand-alone implementation of the SHA256 algorithm. It is designed to - * have as little dependencies as possible so it can be used in the - * kexec_file purgatory. In other cases you should generally use the - * hash APIs from include/crypto/hash.h. Especially when hashing large - * amounts of data as those APIs may be hw-accelerated. - * - * For details see lib/crypto/sha256.c - */ - -static inline void sha256_init(struct sha256_state *sctx) -{ - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; -} -void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len); -void sha256_final(struct sha256_state *sctx, u8 *out); -void sha256(const u8 *data, unsigned int len, u8 *out); - -static inline void sha224_init(struct sha256_state *sctx) -{ - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; -} -void sha224_update(struct sha256_state *sctx, const u8 *data, unsigned int len); -void sha224_final(struct sha256_state *sctx, u8 *out); - -#endif diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h new file mode 100644 index 000000000000..044ecea60ac8 --- /dev/null +++ b/include/crypto/sha1.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common values for SHA-1 algorithms + */ + +#ifndef _CRYPTO_SHA1_H +#define _CRYPTO_SHA1_H + +#include + +#define SHA1_DIGEST_SIZE 20 +#define SHA1_BLOCK_SIZE 64 + +#define SHA1_H0 0x67452301UL +#define SHA1_H1 0xefcdab89UL +#define SHA1_H2 0x98badcfeUL +#define SHA1_H3 0x10325476UL +#define SHA1_H4 0xc3d2e1f0UL + +extern const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE]; + +struct sha1_state { + u32 state[SHA1_DIGEST_SIZE / 4]; + u64 count; + u8 buffer[SHA1_BLOCK_SIZE]; +}; + +struct shash_desc; + +extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len); + +extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash); + +/* + * An implementation of SHA-1's compression function. Don't use in new code! + * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't + * the correct way to hash something with SHA-1 (use crypto_shash instead). + */ +#define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) +#define SHA1_WORKSPACE_WORDS 16 +void sha1_init(__u32 *buf); +void sha1_transform(__u32 *digest, const char *data, __u32 *W); + +#endif /* _CRYPTO_SHA1_H */ diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h index a5d6033efef7..2e0e7c3827d1 100644 --- a/include/crypto/sha1_base.h +++ b/include/crypto/sha1_base.h @@ -9,7 +9,7 @@ #define _CRYPTO_SHA1_BASE_H #include -#include +#include #include #include #include diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h new file mode 100644 index 000000000000..2838f529f31e --- /dev/null +++ b/include/crypto/sha2.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common values for SHA-2 algorithms + */ + +#ifndef _CRYPTO_SHA2_H +#define _CRYPTO_SHA2_H + +#include + +#define SHA224_DIGEST_SIZE 28 +#define SHA224_BLOCK_SIZE 64 + +#define SHA256_DIGEST_SIZE 32 +#define SHA256_BLOCK_SIZE 64 + +#define SHA384_DIGEST_SIZE 48 +#define SHA384_BLOCK_SIZE 128 + +#define SHA512_DIGEST_SIZE 64 +#define SHA512_BLOCK_SIZE 128 + +#define SHA224_H0 0xc1059ed8UL +#define SHA224_H1 0x367cd507UL +#define SHA224_H2 0x3070dd17UL +#define SHA224_H3 0xf70e5939UL +#define SHA224_H4 0xffc00b31UL +#define SHA224_H5 0x68581511UL +#define SHA224_H6 0x64f98fa7UL +#define SHA224_H7 0xbefa4fa4UL + +#define SHA256_H0 0x6a09e667UL +#define SHA256_H1 0xbb67ae85UL +#define SHA256_H2 0x3c6ef372UL +#define SHA256_H3 0xa54ff53aUL +#define SHA256_H4 0x510e527fUL +#define SHA256_H5 0x9b05688cUL +#define SHA256_H6 0x1f83d9abUL +#define SHA256_H7 0x5be0cd19UL + +#define SHA384_H0 0xcbbb9d5dc1059ed8ULL +#define SHA384_H1 0x629a292a367cd507ULL +#define SHA384_H2 0x9159015a3070dd17ULL +#define SHA384_H3 0x152fecd8f70e5939ULL +#define SHA384_H4 0x67332667ffc00b31ULL +#define SHA384_H5 0x8eb44a8768581511ULL +#define SHA384_H6 0xdb0c2e0d64f98fa7ULL +#define SHA384_H7 0x47b5481dbefa4fa4ULL + +#define SHA512_H0 0x6a09e667f3bcc908ULL +#define SHA512_H1 0xbb67ae8584caa73bULL +#define SHA512_H2 0x3c6ef372fe94f82bULL +#define SHA512_H3 0xa54ff53a5f1d36f1ULL +#define SHA512_H4 0x510e527fade682d1ULL +#define SHA512_H5 0x9b05688c2b3e6c1fULL +#define SHA512_H6 0x1f83d9abfb41bd6bULL +#define SHA512_H7 0x5be0cd19137e2179ULL + +extern const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE]; + +extern const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE]; + +extern const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE]; + +extern const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE]; + +struct sha256_state { + u32 state[SHA256_DIGEST_SIZE / 4]; + u64 count; + u8 buf[SHA256_BLOCK_SIZE]; +}; + +struct sha512_state { + u64 state[SHA512_DIGEST_SIZE / 8]; + u64 count[2]; + u8 buf[SHA512_BLOCK_SIZE]; +}; + +struct shash_desc; + +extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len); + +extern int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash); + +extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len); + +extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash); + +/* + * Stand-alone implementation of the SHA256 algorithm. It is designed to + * have as little dependencies as possible so it can be used in the + * kexec_file purgatory. In other cases you should generally use the + * hash APIs from include/crypto/hash.h. Especially when hashing large + * amounts of data as those APIs may be hw-accelerated. + * + * For details see lib/crypto/sha256.c + */ + +static inline void sha256_init(struct sha256_state *sctx) +{ + sctx->state[0] = SHA256_H0; + sctx->state[1] = SHA256_H1; + sctx->state[2] = SHA256_H2; + sctx->state[3] = SHA256_H3; + sctx->state[4] = SHA256_H4; + sctx->state[5] = SHA256_H5; + sctx->state[6] = SHA256_H6; + sctx->state[7] = SHA256_H7; + sctx->count = 0; +} +void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len); +void sha256_final(struct sha256_state *sctx, u8 *out); +void sha256(const u8 *data, unsigned int len, u8 *out); + +static inline void sha224_init(struct sha256_state *sctx) +{ + sctx->state[0] = SHA224_H0; + sctx->state[1] = SHA224_H1; + sctx->state[2] = SHA224_H2; + sctx->state[3] = SHA224_H3; + sctx->state[4] = SHA224_H4; + sctx->state[5] = SHA224_H5; + sctx->state[6] = SHA224_H6; + sctx->state[7] = SHA224_H7; + sctx->count = 0; +} +void sha224_update(struct sha256_state *sctx, const u8 *data, unsigned int len); +void sha224_final(struct sha256_state *sctx, u8 *out); + +#endif /* _CRYPTO_SHA2_H */ diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index 93f9fd21cc06..76173c613058 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -9,7 +9,7 @@ #define _CRYPTO_SHA256_BASE_H #include -#include +#include #include #include #include diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index 93ab73baa38e..b370b3340b16 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -9,7 +9,7 @@ #define _CRYPTO_SHA512_BASE_H #include -#include +#include #include #include #include diff --git a/include/linux/ccp.h b/include/linux/ccp.h index a5dfbaf2470d..868924dec5a1 100644 --- a/include/linux/ccp.h +++ b/include/linux/ccp.h @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include struct ccp_device; struct ccp_cmd; diff --git a/include/linux/filter.h b/include/linux/filter.h index 72d62cbc1578..6c00140538b9 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include diff --git a/include/linux/purgatory.h b/include/linux/purgatory.h index b950e961cfa8..d7dc1559427f 100644 --- a/include/linux/purgatory.h +++ b/include/linux/purgatory.h @@ -3,7 +3,7 @@ #define _LINUX_PURGATORY_H #include -#include +#include #include struct kexec_sha_region { diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 106e4500fd53..4fcfe0b70c4e 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -11,7 +11,7 @@ #include #include -#include +#include /* vmcoreinfo stuff */ unsigned char *vmcoreinfo_data; diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 8798a8183974..4f8efc278aa7 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -42,7 +42,6 @@ #include #include -#include #include "kexec_internal.h" DEFINE_MUTEX(kexec_mutex); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index e21f6b9234f7..b02086d70492 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index cdef37c05972..72a4b0b1df28 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include static const u32 SHA256_K[] = { diff --git a/lib/digsig.c b/lib/digsig.c index e0627c3e53b2..04b5e55ed95f 100644 --- a/lib/digsig.c +++ b/lib/digsig.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/lib/sha1.c b/lib/sha1.c index 49257a915bb6..9bd1935a1472 100644 --- a/lib/sha1.c +++ b/lib/sha1.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include /* diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c index 85dddfe3a2c6..687d95dce085 100644 --- a/net/ipv6/seg6_hmac.c +++ b/net/ipv6/seg6_hmac.c @@ -35,7 +35,6 @@ #include #include -#include #include #include #include diff --git a/net/mptcp/crypto.c b/net/mptcp/crypto.c index 05d398d3fde4..b472dc149856 100644 --- a/net/mptcp/crypto.c +++ b/net/mptcp/crypto.c @@ -21,7 +21,7 @@ */ #include -#include +#include #include #include "protocol.h" diff --git a/net/mptcp/options.c b/net/mptcp/options.c index a044dd43411d..90cd52df99a6 100644 --- a/net/mptcp/options.c +++ b/net/mptcp/options.c @@ -7,7 +7,7 @@ #define pr_fmt(fmt) "MPTCP: " fmt #include -#include +#include #include #include #include "protocol.h" diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index ac4a1fe3550b..b229ae914d76 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 413c803c5208..547425c20e11 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c index 192e531c146f..87432b35d771 100644 --- a/security/keys/encrypted-keys/encrypted.c +++ b/security/keys/encrypted-keys/encrypted.c @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include "encrypted.h" diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c index b9fe02e5f84f..74d82093cbaa 100644 --- a/security/keys/trusted-keys/trusted_tpm1.c +++ b/security/keys/trusted-keys/trusted_tpm1.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/sound/soc/codecs/cros_ec_codec.c b/sound/soc/codecs/cros_ec_codec.c index 28f039adfa13..58894bf47514 100644 --- a/sound/soc/codecs/cros_ec_codec.c +++ b/sound/soc/codecs/cros_ec_codec.c @@ -8,7 +8,7 @@ * EC for audio function. */ -#include +#include #include #include #include -- cgit v1.2.3 From 7c2f5537ca196ddd00639f66f0d58c9f8d265b27 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 20 Nov 2020 17:01:31 +1100 Subject: crypto: mips/octeon - Fix sparse endianness warnings This patch fixes a number of endianness warnings in the mips/octeon code. Signed-off-by: Herbert Xu --- arch/mips/cavium-octeon/crypto/octeon-crypto.h | 2 +- arch/mips/cavium-octeon/crypto/octeon-md5.c | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) (limited to 'arch') diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/cavium-octeon/crypto/octeon-crypto.h index 7315cc307397..cb68f9e284bb 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-crypto.h +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.h @@ -41,7 +41,7 @@ do { \ */ #define read_octeon_64bit_hash_dword(index) \ ({ \ - u64 __value; \ + __be64 __value; \ \ __asm__ __volatile__ ( \ "dmfc2 %[rt],0x0048+" STR(index) \ diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium-octeon/crypto/octeon-md5.c index 8c8ea139653e..5ee4ade99b99 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-md5.c +++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c @@ -68,10 +68,11 @@ static int octeon_md5_init(struct shash_desc *desc) { struct md5_state *mctx = shash_desc_ctx(desc); - mctx->hash[0] = cpu_to_le32(MD5_H0); - mctx->hash[1] = cpu_to_le32(MD5_H1); - mctx->hash[2] = cpu_to_le32(MD5_H2); - mctx->hash[3] = cpu_to_le32(MD5_H3); + mctx->hash[0] = MD5_H0; + mctx->hash[1] = MD5_H1; + mctx->hash[2] = MD5_H2; + mctx->hash[3] = MD5_H3; + cpu_to_le32_array(mctx->hash, 4); mctx->byte_count = 0; return 0; @@ -139,8 +140,9 @@ static int octeon_md5_final(struct shash_desc *desc, u8 *out) } memset(p, 0, padding); - mctx->block[14] = cpu_to_le32(mctx->byte_count << 3); - mctx->block[15] = cpu_to_le32(mctx->byte_count >> 29); + mctx->block[14] = mctx->byte_count << 3; + mctx->block[15] = mctx->byte_count >> 29; + cpu_to_le32_array(mctx->block + 14, 2); octeon_md5_transform(mctx->block); octeon_md5_read_hash(mctx); -- cgit v1.2.3 From e547655238f70a7b238e14f18d968428ff6b6b93 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 20 Nov 2020 17:28:38 +1100 Subject: crypto: powerpc/sha256-spe - Fix sparse endianness warning This patch fixes a sparse endianness warning in sha256-spe. Signed-off-by: Herbert Xu --- arch/powerpc/crypto/sha256-spe-glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index 88530ae0791f..a6e650a97d8f 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -177,7 +177,7 @@ static int ppc_spe_sha256_final(struct shash_desc *desc, u8 *out) static int ppc_spe_sha224_final(struct shash_desc *desc, u8 *out) { - u32 D[SHA256_DIGEST_SIZE >> 2]; + __be32 D[SHA256_DIGEST_SIZE >> 2]; __be32 *dst = (__be32 *)out; ppc_spe_sha256_final(desc, (u8 *)D); -- cgit v1.2.3 From 6cf1a144d3f5752400cad99d9142fba668cc4ddb Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 20 Nov 2020 17:41:26 +1100 Subject: crypto: sparc - Fix sparse endianness warnings This patch fixes a coulpe of sparse endianness warnings. Signed-off-by: Herbert Xu --- arch/sparc/crypto/crc32c_glue.c | 2 +- arch/sparc/crypto/md5_glue.c | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'arch') diff --git a/arch/sparc/crypto/crc32c_glue.c b/arch/sparc/crypto/crc32c_glue.c index 4e9323229e71..82efb7f81c28 100644 --- a/arch/sparc/crypto/crc32c_glue.c +++ b/arch/sparc/crypto/crc32c_glue.c @@ -35,7 +35,7 @@ static int crc32c_sparc64_setkey(struct crypto_shash *hash, const u8 *key, if (keylen != sizeof(u32)) return -EINVAL; - *(__le32 *)mctx = le32_to_cpup((__le32 *)key); + *mctx = le32_to_cpup((__le32 *)key); return 0; } diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c index 111283fe837e..511db98d590a 100644 --- a/arch/sparc/crypto/md5_glue.c +++ b/arch/sparc/crypto/md5_glue.c @@ -33,10 +33,11 @@ static int md5_sparc64_init(struct shash_desc *desc) { struct md5_state *mctx = shash_desc_ctx(desc); - mctx->hash[0] = cpu_to_le32(MD5_H0); - mctx->hash[1] = cpu_to_le32(MD5_H1); - mctx->hash[2] = cpu_to_le32(MD5_H2); - mctx->hash[3] = cpu_to_le32(MD5_H3); + mctx->hash[0] = MD5_H0; + mctx->hash[1] = MD5_H1; + mctx->hash[2] = MD5_H2; + mctx->hash[3] = MD5_H3; + le32_to_cpu_array(mctx->hash, 4); mctx->byte_count = 0; return 0; -- cgit v1.2.3 From 56c6da16c3631f953fb20d8b7ddccdf493377ad4 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 20 Nov 2020 17:02:32 +0800 Subject: crypto: hisilicon/trng - add HiSilicon TRNG driver support Move existing char/hw_random/hisi-trng-v2.c to crypto/hisilicon/trng.c. Signed-off-by: Weili Qian Reviewed-by: Zaibo Xu Signed-off-by: Herbert Xu --- arch/arm64/configs/defconfig | 1 + drivers/crypto/hisilicon/Kconfig | 7 +++ drivers/crypto/hisilicon/Makefile | 1 + drivers/crypto/hisilicon/trng/Makefile | 2 + drivers/crypto/hisilicon/trng/trng.c | 99 ++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+) create mode 100644 drivers/crypto/hisilicon/trng/Makefile create mode 100644 drivers/crypto/hisilicon/trng/trng.c (limited to 'arch') diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 17a2df6a263e..646c3b41c60a 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1081,6 +1081,7 @@ CONFIG_CRYPTO_DEV_CCREE=m CONFIG_CRYPTO_DEV_HISI_SEC2=m CONFIG_CRYPTO_DEV_HISI_ZIP=m CONFIG_CRYPTO_DEV_HISI_HPRE=m +CONFIG_CRYPTO_DEV_HISI_TRNG=m CONFIG_CMA_SIZE_MBYTES=32 CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 9c3b3ca815e6..2cd3298920fa 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -71,3 +71,10 @@ config CRYPTO_DEV_HISI_HPRE help Support for HiSilicon HPRE(High Performance RSA Engine) accelerator, which can accelerate RSA and DH algorithms. + +config CRYPTO_DEV_HISI_TRNG + tristate "Support for HISI TRNG Driver" + depends on ARM64 && ACPI + select HW_RANDOM + help + Support for HiSilicon TRNG Driver. diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile index 7f5f74c72baa..1e89269a2e4b 100644 --- a/drivers/crypto/hisilicon/Makefile +++ b/drivers/crypto/hisilicon/Makefile @@ -5,3 +5,4 @@ obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += sec2/ obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += hisi_qm.o hisi_qm-objs = qm.o sgl.o obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/ +obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += trng/ diff --git a/drivers/crypto/hisilicon/trng/Makefile b/drivers/crypto/hisilicon/trng/Makefile new file mode 100644 index 000000000000..d909079f351c --- /dev/null +++ b/drivers/crypto/hisilicon/trng/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += hisi-trng-v2.o +hisi-trng-v2-objs = trng.o diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c new file mode 100644 index 000000000000..6a65b8232ce0 --- /dev/null +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HISI_TRNG_REG 0x00F0 +#define HISI_TRNG_BYTES 4 +#define HISI_TRNG_QUALITY 512 +#define SLEEP_US 10 +#define TIMEOUT_US 10000 + +struct hisi_trng { + void __iomem *base; + struct hwrng rng; +}; + +static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct hisi_trng *trng; + int currsize = 0; + u32 val = 0; + u32 ret; + + trng = container_of(rng, struct hisi_trng, rng); + + do { + ret = readl_poll_timeout(trng->base + HISI_TRNG_REG, val, + val, SLEEP_US, TIMEOUT_US); + if (ret) + return currsize; + + if (max - currsize >= HISI_TRNG_BYTES) { + memcpy(buf + currsize, &val, HISI_TRNG_BYTES); + currsize += HISI_TRNG_BYTES; + if (currsize == max) + return currsize; + continue; + } + + /* copy remaining bytes */ + memcpy(buf + currsize, &val, max - currsize); + currsize = max; + } while (currsize < max); + + return currsize; +} + +static int hisi_trng_probe(struct platform_device *pdev) +{ + struct hisi_trng *trng; + int ret; + + trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL); + if (!trng) + return -ENOMEM; + + trng->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(trng->base)) + return PTR_ERR(trng->base); + + trng->rng.name = pdev->name; + trng->rng.read = hisi_trng_read; + trng->rng.quality = HISI_TRNG_QUALITY; + + ret = devm_hwrng_register(&pdev->dev, &trng->rng); + if (ret) + dev_err(&pdev->dev, "failed to register hwrng!\n"); + + return ret; +} + +static const struct acpi_device_id hisi_trng_acpi_match[] = { + { "HISI02B3", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); + +static struct platform_driver hisi_trng_driver = { + .probe = hisi_trng_probe, + .driver = { + .name = "hisi-trng-v2", + .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), + }, +}; + +module_platform_driver(hisi_trng_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Weili Qian "); +MODULE_AUTHOR("Zaibo Xu "); +MODULE_DESCRIPTION("HiSilicon true random number generator V2 driver"); -- cgit v1.2.3 From f3456b9fd269c6d0c973b136c5449d46b2510f4b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 26 Nov 2020 08:49:07 +0100 Subject: crypto: arm/aes-ce - work around Cortex-A57/A72 silion errata ARM Cortex-A57 and Cortex-A72 cores running in 32-bit mode are affected by silicon errata #1742098 and #1655431, respectively, where the second instruction of a AES instruction pair may execute twice if an interrupt is taken right after the first instruction consumes an input register of which a single 32-bit lane has been updated the last time it was modified. This is not such a rare occurrence as it may seem: in counter mode, only the least significant 32-bit word is incremented in the absence of a carry, which makes our counter mode implementation susceptible to these errata. So let's shuffle the counter assignments around a bit so that the most recent updates when the AES instruction pair executes are 128-bit wide. [0] ARM-EPM-049219 v23 Cortex-A57 MPCore Software Developers Errata Notice [1] ARM-EPM-012079 v11.0 Cortex-A72 MPCore Software Developers Errata Notice Cc: # v5.4+ Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-ce-core.S | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) (limited to 'arch') diff --git a/arch/arm/crypto/aes-ce-core.S b/arch/arm/crypto/aes-ce-core.S index 4d1707388d94..312428d83eed 100644 --- a/arch/arm/crypto/aes-ce-core.S +++ b/arch/arm/crypto/aes-ce-core.S @@ -386,20 +386,32 @@ ENTRY(ce_aes_ctr_encrypt) .Lctrloop4x: subs r4, r4, #4 bmi .Lctr1x - add r6, r6, #1 + + /* + * NOTE: the sequence below has been carefully tweaked to avoid + * a silicon erratum that exists in Cortex-A57 (#1742098) and + * Cortex-A72 (#1655431) cores, where AESE/AESMC instruction pairs + * may produce an incorrect result if they take their input from a + * register of which a single 32-bit lane has been updated the last + * time it was modified. To work around this, the lanes of registers + * q0-q3 below are not manipulated individually, and the different + * counter values are prepared by successive manipulations of q7. + */ + add ip, r6, #1 vmov q0, q7 + rev ip, ip + add lr, r6, #2 + vmov s31, ip @ set lane 3 of q1 via q7 + add ip, r6, #3 + rev lr, lr vmov q1, q7 - rev ip, r6 - add r6, r6, #1 + vmov s31, lr @ set lane 3 of q2 via q7 + rev ip, ip vmov q2, q7 - vmov s7, ip - rev ip, r6 - add r6, r6, #1 + vmov s31, ip @ set lane 3 of q3 via q7 + add r6, r6, #4 vmov q3, q7 - vmov s11, ip - rev ip, r6 - add r6, r6, #1 - vmov s15, ip + vld1.8 {q4-q5}, [r1]! vld1.8 {q6}, [r1]! vld1.8 {q15}, [r1]! -- cgit v1.2.3 From 032d049ea0f45b45c21f3f02b542aa18bc6b6428 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 27 Nov 2020 10:44:52 +0100 Subject: crypto: aesni - Use TEST %reg,%reg instead of CMP $0,%reg CMP $0,%reg can't set overflow flag, so we can use shorter TEST %reg,%reg instruction when only zero and sign flags are checked (E,L,LE,G,GE conditions). Signed-off-by: Uros Bizjak Cc: Herbert Xu Cc: Borislav Petkov Cc: "H. Peter Anvin" Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_asm.S | 20 ++++++++++---------- arch/x86/crypto/aesni-intel_avx-x86_64.S | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) (limited to 'arch') diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 1852b19a73a0..d1436c37008b 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -318,7 +318,7 @@ _initial_blocks_\@: # Main loop - Encrypt/Decrypt remaining blocks - cmp $0, %r13 + test %r13, %r13 je _zero_cipher_left_\@ sub $64, %r13 je _four_cipher_left_\@ @@ -437,7 +437,7 @@ _multiple_of_16_bytes_\@: mov PBlockLen(%arg2), %r12 - cmp $0, %r12 + test %r12, %r12 je _partial_done\@ GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 @@ -474,7 +474,7 @@ _T_8_\@: add $8, %r10 sub $8, %r11 psrldq $8, %xmm0 - cmp $0, %r11 + test %r11, %r11 je _return_T_done_\@ _T_4_\@: movd %xmm0, %eax @@ -482,7 +482,7 @@ _T_4_\@: add $4, %r10 sub $4, %r11 psrldq $4, %xmm0 - cmp $0, %r11 + test %r11, %r11 je _return_T_done_\@ _T_123_\@: movd %xmm0, %eax @@ -619,7 +619,7 @@ _get_AAD_blocks\@: /* read the last <16B of AAD */ _get_AAD_rest\@: - cmp $0, %r11 + test %r11, %r11 je _get_AAD_done\@ READ_PARTIAL_BLOCK %r10, %r11, \TMP1, \TMP7 @@ -640,7 +640,7 @@ _get_AAD_done\@: .macro PARTIAL_BLOCK CYPH_PLAIN_OUT PLAIN_CYPH_IN PLAIN_CYPH_LEN DATA_OFFSET \ AAD_HASH operation mov PBlockLen(%arg2), %r13 - cmp $0, %r13 + test %r13, %r13 je _partial_block_done_\@ # Leave Macro if no partial blocks # Read in input data without over reading cmp $16, \PLAIN_CYPH_LEN @@ -692,7 +692,7 @@ _no_extra_mask_1_\@: pshufb %xmm2, %xmm3 pxor %xmm3, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_1_\@ # GHASH computation for the last <16 Byte block @@ -727,7 +727,7 @@ _no_extra_mask_2_\@: pshufb %xmm2, %xmm9 pxor %xmm9, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_2_\@ # GHASH computation for the last <16 Byte block @@ -747,7 +747,7 @@ _encode_done_\@: pshufb %xmm2, %xmm9 .endif # output encrypted Bytes - cmp $0, %r10 + test %r10, %r10 jl _partial_fill_\@ mov %r13, %r12 mov $16, %r13 @@ -2720,7 +2720,7 @@ SYM_FUNC_END(aesni_ctr_enc) */ SYM_FUNC_START(aesni_xts_crypt8) FRAME_BEGIN - cmpb $0, %cl + testb %cl, %cl movl $0, %ecx movl $240, %r10d leaq _aesni_enc4, %r11 diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S index 5fee47956f3b..2cf8e94d986a 100644 --- a/arch/x86/crypto/aesni-intel_avx-x86_64.S +++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S @@ -369,7 +369,7 @@ _initial_num_blocks_is_0\@: _initial_blocks_encrypted\@: - cmp $0, %r13 + test %r13, %r13 je _zero_cipher_left\@ sub $128, %r13 @@ -528,7 +528,7 @@ _multiple_of_16_bytes\@: vmovdqu HashKey(arg2), %xmm13 mov PBlockLen(arg2), %r12 - cmp $0, %r12 + test %r12, %r12 je _partial_done\@ #GHASH computation for the last <16 Byte block @@ -573,7 +573,7 @@ _T_8\@: add $8, %r10 sub $8, %r11 vpsrldq $8, %xmm9, %xmm9 - cmp $0, %r11 + test %r11, %r11 je _return_T_done\@ _T_4\@: vmovd %xmm9, %eax @@ -581,7 +581,7 @@ _T_4\@: add $4, %r10 sub $4, %r11 vpsrldq $4, %xmm9, %xmm9 - cmp $0, %r11 + test %r11, %r11 je _return_T_done\@ _T_123\@: vmovd %xmm9, %eax @@ -625,7 +625,7 @@ _get_AAD_blocks\@: cmp $16, %r11 jge _get_AAD_blocks\@ vmovdqu \T8, \T7 - cmp $0, %r11 + test %r11, %r11 je _get_AAD_done\@ vpxor \T7, \T7, \T7 @@ -644,7 +644,7 @@ _get_AAD_rest8\@: vpxor \T1, \T7, \T7 jmp _get_AAD_rest8\@ _get_AAD_rest4\@: - cmp $0, %r11 + test %r11, %r11 jle _get_AAD_rest0\@ mov (%r10), %eax movq %rax, \T1 @@ -749,7 +749,7 @@ _done_read_partial_block_\@: .macro PARTIAL_BLOCK GHASH_MUL CYPH_PLAIN_OUT PLAIN_CYPH_IN PLAIN_CYPH_LEN DATA_OFFSET \ AAD_HASH ENC_DEC mov PBlockLen(arg2), %r13 - cmp $0, %r13 + test %r13, %r13 je _partial_block_done_\@ # Leave Macro if no partial blocks # Read in input data without over reading cmp $16, \PLAIN_CYPH_LEN @@ -801,7 +801,7 @@ _no_extra_mask_1_\@: vpshufb %xmm2, %xmm3, %xmm3 vpxor %xmm3, \AAD_HASH, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_1_\@ # GHASH computation for the last <16 Byte block @@ -836,7 +836,7 @@ _no_extra_mask_2_\@: vpshufb %xmm2, %xmm9, %xmm9 vpxor %xmm9, \AAD_HASH, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_2_\@ # GHASH computation for the last <16 Byte block @@ -856,7 +856,7 @@ _encode_done_\@: vpshufb %xmm2, %xmm9, %xmm9 .endif # output encrypted Bytes - cmp $0, %r10 + test %r10, %r10 jl _partial_fill_\@ mov %r13, %r12 mov $16, %r13 -- cgit v1.2.3 From 0b837f1ef8822a8df880ffba6778ba42f707b83d Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 27 Nov 2020 10:59:43 +0100 Subject: crypto: x86/sha512 - Use TEST %reg,%reg instead of CMP $0,%reg CMP $0,%reg can't set overflow flag, so we can use shorter TEST %reg,%reg instruction when only zero and sign flags are checked (E,L,LE,G,GE conditions). Signed-off-by: Uros Bizjak Cc: Herbert Xu Cc: Borislav Petkov Cc: "H. Peter Anvin" Signed-off-by: Herbert Xu --- arch/x86/crypto/sha512-avx-asm.S | 2 +- arch/x86/crypto/sha512-ssse3-asm.S | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/x86/crypto/sha512-avx-asm.S b/arch/x86/crypto/sha512-avx-asm.S index 63470fd6ae32..684d58c8bc4f 100644 --- a/arch/x86/crypto/sha512-avx-asm.S +++ b/arch/x86/crypto/sha512-avx-asm.S @@ -278,7 +278,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE # "blocks" is the message length in SHA512 blocks ######################################################################## SYM_FUNC_START(sha512_transform_avx) - cmp $0, msglen + test msglen, msglen je nowork # Allocate Stack Space diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/arch/x86/crypto/sha512-ssse3-asm.S index 7946a1bee85b..50812af0b083 100644 --- a/arch/x86/crypto/sha512-ssse3-asm.S +++ b/arch/x86/crypto/sha512-ssse3-asm.S @@ -280,7 +280,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE ######################################################################## SYM_FUNC_START(sha512_transform_ssse3) - cmp $0, msglen + test msglen, msglen je nowork # Allocate Stack Space -- cgit v1.2.3 From be169fe3cec9efe2bfff98b3f645bca6cc7d09cd Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 27 Nov 2020 11:18:12 +0100 Subject: crypto: x86/poly1305 - Use TEST %reg,%reg instead of CMP $0,%reg CMP $0,%reg can't set overflow flag, so we can use shorter TEST %reg,%reg instruction when only zero and sign flags are checked (E,L,LE,G,GE conditions). Signed-off-by: Uros Bizjak Cc: Herbert Xu Cc: Borislav Petkov Cc: "H. Peter Anvin" Signed-off-by: Herbert Xu --- arch/x86/crypto/poly1305-x86_64-cryptogams.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl index 7d568012cc15..71fae5a09e56 100644 --- a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl +++ b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl @@ -251,7 +251,7 @@ $code.=<<___; mov %rax,8($ctx) mov %rax,16($ctx) - cmp \$0,$inp + test $inp,$inp je .Lno_key ___ $code.=<<___ if (!$kernel); -- cgit v1.2.3