diff options
Diffstat (limited to 'drivers/crypto/intel')
38 files changed, 1781 insertions, 421 deletions
diff --git a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c index ae31be00357a..1e2fd9a754ec 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c @@ -5,24 +5,23 @@ * Copyright (C) 2018-2020 Intel Corporation */ +#include <crypto/aes.h> +#include <crypto/engine.h> +#include <crypto/gcm.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/skcipher.h> +#include <crypto/scatterwalk.h> #include <linux/clk.h> #include <linux/completion.h> -#include <linux/crypto.h> #include <linux/dma-mapping.h> +#include <linux/err.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> -#include <linux/types.h> - -#include <crypto/aes.h> -#include <crypto/engine.h> -#include <crypto/gcm.h> -#include <crypto/scatterwalk.h> - -#include <crypto/internal/aead.h> -#include <crypto/internal/skcipher.h> +#include <linux/string.h> #include "ocs-aes.h" @@ -38,7 +37,6 @@ /** * struct ocs_aes_tctx - OCS AES Transform context - * @engine_ctx: Engine context. * @aes_dev: The OCS AES device. * @key: AES/SM4 key. * @key_len: The length (in bytes) of @key. @@ -47,7 +45,6 @@ * @use_fallback: Whether or not fallback cipher should be used. */ struct ocs_aes_tctx { - struct crypto_engine_ctx engine_ctx; struct ocs_aes_dev *aes_dev; u8 key[OCS_AES_KEYSIZE_256]; unsigned int key_len; @@ -1148,15 +1145,6 @@ static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req) return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM); } -static inline int ocs_common_init(struct ocs_aes_tctx *tctx) -{ - tctx->engine_ctx.op.prepare_request = NULL; - tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_sk_do_one_request; - tctx->engine_ctx.op.unprepare_request = NULL; - - return 0; -} - static int ocs_aes_init_tfm(struct crypto_skcipher *tfm) { const char *alg_name = crypto_tfm_alg_name(&tfm->base); @@ -1172,16 +1160,14 @@ static int ocs_aes_init_tfm(struct crypto_skcipher *tfm) crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); - return ocs_common_init(tctx); + return 0; } static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm) { - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); - return ocs_common_init(tctx); + return 0; } static inline void clear_key(struct ocs_aes_tctx *tctx) @@ -1206,15 +1192,6 @@ static void ocs_exit_tfm(struct crypto_skcipher *tfm) } } -static inline int ocs_common_aead_init(struct ocs_aes_tctx *tctx) -{ - tctx->engine_ctx.op.prepare_request = NULL; - tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_aead_do_one_request; - tctx->engine_ctx.op.unprepare_request = NULL; - - return 0; -} - static int ocs_aes_aead_cra_init(struct crypto_aead *tfm) { const char *alg_name = crypto_tfm_alg_name(&tfm->base); @@ -1233,7 +1210,7 @@ static int ocs_aes_aead_cra_init(struct crypto_aead *tfm) (sizeof(struct aead_request) + crypto_aead_reqsize(tctx->sw_cipher.aead)))); - return ocs_common_aead_init(tctx); + return 0; } static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm, @@ -1261,11 +1238,9 @@ static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm, static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm) { - struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); - crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); - return ocs_common_aead_init(tctx); + return 0; } static void ocs_aead_cra_exit(struct crypto_aead *tfm) @@ -1280,182 +1255,190 @@ static void ocs_aead_cra_exit(struct crypto_aead *tfm) } } -static struct skcipher_alg algs[] = { +static struct skcipher_engine_alg algs[] = { #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB { - .base.cra_name = "ecb(aes)", - .base.cra_driver_name = "ecb-aes-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_AES_MIN_KEY_SIZE, - .max_keysize = OCS_AES_MAX_KEY_SIZE, - .setkey = kmb_ocs_aes_set_key, - .encrypt = kmb_ocs_aes_ecb_encrypt, - .decrypt = kmb_ocs_aes_ecb_decrypt, - .init = ocs_aes_init_tfm, - .exit = ocs_exit_tfm, + .base.base.cra_name = "ecb(aes)", + .base.base.cra_driver_name = "ecb-aes-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_AES_MIN_KEY_SIZE, + .base.max_keysize = OCS_AES_MAX_KEY_SIZE, + .base.setkey = kmb_ocs_aes_set_key, + .base.encrypt = kmb_ocs_aes_ecb_encrypt, + .base.decrypt = kmb_ocs_aes_ecb_decrypt, + .base.init = ocs_aes_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, }, #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ { - .base.cra_name = "cbc(aes)", - .base.cra_driver_name = "cbc-aes-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_AES_MIN_KEY_SIZE, - .max_keysize = OCS_AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_aes_set_key, - .encrypt = kmb_ocs_aes_cbc_encrypt, - .decrypt = kmb_ocs_aes_cbc_decrypt, - .init = ocs_aes_init_tfm, - .exit = ocs_exit_tfm, + .base.base.cra_name = "cbc(aes)", + .base.base.cra_driver_name = "cbc-aes-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_AES_MIN_KEY_SIZE, + .base.max_keysize = OCS_AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_aes_set_key, + .base.encrypt = kmb_ocs_aes_cbc_encrypt, + .base.decrypt = kmb_ocs_aes_cbc_decrypt, + .base.init = ocs_aes_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, }, { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_AES_MIN_KEY_SIZE, - .max_keysize = OCS_AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_aes_set_key, - .encrypt = kmb_ocs_aes_ctr_encrypt, - .decrypt = kmb_ocs_aes_ctr_decrypt, - .init = ocs_aes_init_tfm, - .exit = ocs_exit_tfm, + .base.base.cra_name = "ctr(aes)", + .base.base.cra_driver_name = "ctr-aes-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.base.cra_blocksize = 1, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_AES_MIN_KEY_SIZE, + .base.max_keysize = OCS_AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_aes_set_key, + .base.encrypt = kmb_ocs_aes_ctr_encrypt, + .base.decrypt = kmb_ocs_aes_ctr_decrypt, + .base.init = ocs_aes_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, }, #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS { - .base.cra_name = "cts(cbc(aes))", - .base.cra_driver_name = "cts-aes-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_AES_MIN_KEY_SIZE, - .max_keysize = OCS_AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_aes_set_key, - .encrypt = kmb_ocs_aes_cts_encrypt, - .decrypt = kmb_ocs_aes_cts_decrypt, - .init = ocs_aes_init_tfm, - .exit = ocs_exit_tfm, + .base.base.cra_name = "cts(cbc(aes))", + .base.base.cra_driver_name = "cts-aes-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_AES_MIN_KEY_SIZE, + .base.max_keysize = OCS_AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_aes_set_key, + .base.encrypt = kmb_ocs_aes_cts_encrypt, + .base.decrypt = kmb_ocs_aes_cts_decrypt, + .base.init = ocs_aes_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, }, #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB { - .base.cra_name = "ecb(sm4)", - .base.cra_driver_name = "ecb-sm4-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_SM4_KEY_SIZE, - .max_keysize = OCS_SM4_KEY_SIZE, - .setkey = kmb_ocs_sm4_set_key, - .encrypt = kmb_ocs_sm4_ecb_encrypt, - .decrypt = kmb_ocs_sm4_ecb_decrypt, - .init = ocs_sm4_init_tfm, - .exit = ocs_exit_tfm, + .base.base.cra_name = "ecb(sm4)", + .base.base.cra_driver_name = "ecb-sm4-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_SM4_KEY_SIZE, + .base.max_keysize = OCS_SM4_KEY_SIZE, + .base.setkey = kmb_ocs_sm4_set_key, + .base.encrypt = kmb_ocs_sm4_ecb_encrypt, + .base.decrypt = kmb_ocs_sm4_ecb_decrypt, + .base.init = ocs_sm4_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, }, #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ { - .base.cra_name = "cbc(sm4)", - .base.cra_driver_name = "cbc-sm4-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_SM4_KEY_SIZE, - .max_keysize = OCS_SM4_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_sm4_set_key, - .encrypt = kmb_ocs_sm4_cbc_encrypt, - .decrypt = kmb_ocs_sm4_cbc_decrypt, - .init = ocs_sm4_init_tfm, - .exit = ocs_exit_tfm, + .base.base.cra_name = "cbc(sm4)", + .base.base.cra_driver_name = "cbc-sm4-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_SM4_KEY_SIZE, + .base.max_keysize = OCS_SM4_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_sm4_set_key, + .base.encrypt = kmb_ocs_sm4_cbc_encrypt, + .base.decrypt = kmb_ocs_sm4_cbc_decrypt, + .base.init = ocs_sm4_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, }, { - .base.cra_name = "ctr(sm4)", - .base.cra_driver_name = "ctr-sm4-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_SM4_KEY_SIZE, - .max_keysize = OCS_SM4_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_sm4_set_key, - .encrypt = kmb_ocs_sm4_ctr_encrypt, - .decrypt = kmb_ocs_sm4_ctr_decrypt, - .init = ocs_sm4_init_tfm, - .exit = ocs_exit_tfm, + .base.base.cra_name = "ctr(sm4)", + .base.base.cra_driver_name = "ctr-sm4-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.base.cra_blocksize = 1, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_SM4_KEY_SIZE, + .base.max_keysize = OCS_SM4_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_sm4_set_key, + .base.encrypt = kmb_ocs_sm4_ctr_encrypt, + .base.decrypt = kmb_ocs_sm4_ctr_decrypt, + .base.init = ocs_sm4_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, }, #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS { - .base.cra_name = "cts(cbc(sm4))", - .base.cra_driver_name = "cts-sm4-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_SM4_KEY_SIZE, - .max_keysize = OCS_SM4_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_sm4_set_key, - .encrypt = kmb_ocs_sm4_cts_encrypt, - .decrypt = kmb_ocs_sm4_cts_decrypt, - .init = ocs_sm4_init_tfm, - .exit = ocs_exit_tfm, + .base.base.cra_name = "cts(cbc(sm4))", + .base.base.cra_driver_name = "cts-sm4-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_SM4_KEY_SIZE, + .base.max_keysize = OCS_SM4_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_sm4_set_key, + .base.encrypt = kmb_ocs_sm4_cts_encrypt, + .base.decrypt = kmb_ocs_sm4_cts_decrypt, + .base.init = ocs_sm4_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, } #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ }; -static struct aead_alg algs_aead[] = { +static struct aead_engine_alg algs_aead[] = { { - .base = { + .base.base = { .cra_name = "gcm(aes)", .cra_driver_name = "gcm-aes-keembay-ocs", .cra_priority = KMB_OCS_PRIORITY, @@ -1467,17 +1450,18 @@ static struct aead_alg algs_aead[] = { .cra_alignmask = 0, .cra_module = THIS_MODULE, }, - .init = ocs_aes_aead_cra_init, - .exit = ocs_aead_cra_exit, - .ivsize = GCM_AES_IV_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setauthsize = kmb_ocs_aead_gcm_setauthsize, - .setkey = kmb_ocs_aes_aead_set_key, - .encrypt = kmb_ocs_aes_gcm_encrypt, - .decrypt = kmb_ocs_aes_gcm_decrypt, + .base.init = ocs_aes_aead_cra_init, + .base.exit = ocs_aead_cra_exit, + .base.ivsize = GCM_AES_IV_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.setauthsize = kmb_ocs_aead_gcm_setauthsize, + .base.setkey = kmb_ocs_aes_aead_set_key, + .base.encrypt = kmb_ocs_aes_gcm_encrypt, + .base.decrypt = kmb_ocs_aes_gcm_decrypt, + .op.do_one_request = kmb_ocs_aes_aead_do_one_request, }, { - .base = { + .base.base = { .cra_name = "ccm(aes)", .cra_driver_name = "ccm-aes-keembay-ocs", .cra_priority = KMB_OCS_PRIORITY, @@ -1489,17 +1473,18 @@ static struct aead_alg algs_aead[] = { .cra_alignmask = 0, .cra_module = THIS_MODULE, }, - .init = ocs_aes_aead_cra_init, - .exit = ocs_aead_cra_exit, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setauthsize = kmb_ocs_aead_ccm_setauthsize, - .setkey = kmb_ocs_aes_aead_set_key, - .encrypt = kmb_ocs_aes_ccm_encrypt, - .decrypt = kmb_ocs_aes_ccm_decrypt, + .base.init = ocs_aes_aead_cra_init, + .base.exit = ocs_aead_cra_exit, + .base.ivsize = AES_BLOCK_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.setauthsize = kmb_ocs_aead_ccm_setauthsize, + .base.setkey = kmb_ocs_aes_aead_set_key, + .base.encrypt = kmb_ocs_aes_ccm_encrypt, + .base.decrypt = kmb_ocs_aes_ccm_decrypt, + .op.do_one_request = kmb_ocs_aes_aead_do_one_request, }, { - .base = { + .base.base = { .cra_name = "gcm(sm4)", .cra_driver_name = "gcm-sm4-keembay-ocs", .cra_priority = KMB_OCS_PRIORITY, @@ -1510,17 +1495,18 @@ static struct aead_alg algs_aead[] = { .cra_alignmask = 0, .cra_module = THIS_MODULE, }, - .init = ocs_sm4_aead_cra_init, - .exit = ocs_aead_cra_exit, - .ivsize = GCM_AES_IV_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setauthsize = kmb_ocs_aead_gcm_setauthsize, - .setkey = kmb_ocs_sm4_aead_set_key, - .encrypt = kmb_ocs_sm4_gcm_encrypt, - .decrypt = kmb_ocs_sm4_gcm_decrypt, + .base.init = ocs_sm4_aead_cra_init, + .base.exit = ocs_aead_cra_exit, + .base.ivsize = GCM_AES_IV_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.setauthsize = kmb_ocs_aead_gcm_setauthsize, + .base.setkey = kmb_ocs_sm4_aead_set_key, + .base.encrypt = kmb_ocs_sm4_gcm_encrypt, + .base.decrypt = kmb_ocs_sm4_gcm_decrypt, + .op.do_one_request = kmb_ocs_aes_aead_do_one_request, }, { - .base = { + .base.base = { .cra_name = "ccm(sm4)", .cra_driver_name = "ccm-sm4-keembay-ocs", .cra_priority = KMB_OCS_PRIORITY, @@ -1531,21 +1517,22 @@ static struct aead_alg algs_aead[] = { .cra_alignmask = 0, .cra_module = THIS_MODULE, }, - .init = ocs_sm4_aead_cra_init, - .exit = ocs_aead_cra_exit, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setauthsize = kmb_ocs_aead_ccm_setauthsize, - .setkey = kmb_ocs_sm4_aead_set_key, - .encrypt = kmb_ocs_sm4_ccm_encrypt, - .decrypt = kmb_ocs_sm4_ccm_decrypt, + .base.init = ocs_sm4_aead_cra_init, + .base.exit = ocs_aead_cra_exit, + .base.ivsize = AES_BLOCK_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.setauthsize = kmb_ocs_aead_ccm_setauthsize, + .base.setkey = kmb_ocs_sm4_aead_set_key, + .base.encrypt = kmb_ocs_sm4_ccm_encrypt, + .base.decrypt = kmb_ocs_sm4_ccm_decrypt, + .op.do_one_request = kmb_ocs_aes_aead_do_one_request, } }; static void unregister_aes_algs(struct ocs_aes_dev *aes_dev) { - crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); + crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + crypto_engine_unregister_skciphers(algs, ARRAY_SIZE(algs)); } static int register_aes_algs(struct ocs_aes_dev *aes_dev) @@ -1556,13 +1543,13 @@ static int register_aes_algs(struct ocs_aes_dev *aes_dev) * If any algorithm fails to register, all preceding algorithms that * were successfully registered will be automatically unregistered. */ - ret = crypto_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + ret = crypto_engine_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); if (ret) return ret; - ret = crypto_register_skciphers(algs, ARRAY_SIZE(algs)); + ret = crypto_engine_register_skciphers(algs, ARRAY_SIZE(algs)); if (ret) - crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); + crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); return ret; } diff --git a/drivers/crypto/intel/keembay/keembay-ocs-ecc.c b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c index 2269df17514c..fb95deed9057 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-ecc.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c @@ -7,30 +7,27 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <crypto/ecc_curve.h> +#include <crypto/ecdh.h> +#include <crypto/engine.h> +#include <crypto/internal/ecc.h> +#include <crypto/internal/kpp.h> +#include <crypto/kpp.h> +#include <crypto/rng.h> #include <linux/clk.h> #include <linux/completion.h> -#include <linux/crypto.h> -#include <linux/delay.h> +#include <linux/err.h> #include <linux/fips.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iopoll.h> #include <linux/irq.h> +#include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> #include <linux/scatterlist.h> -#include <linux/slab.h> -#include <linux/types.h> - -#include <crypto/ecc_curve.h> -#include <crypto/ecdh.h> -#include <crypto/engine.h> -#include <crypto/kpp.h> -#include <crypto/rng.h> - -#include <crypto/internal/ecc.h> -#include <crypto/internal/kpp.h> +#include <linux/string.h> #define DRV_NAME "keembay-ocs-ecc" @@ -95,13 +92,11 @@ struct ocs_ecc_dev { /** * struct ocs_ecc_ctx - Transformation context. - * @engine_ctx: Crypto engine ctx. * @ecc_dev: The ECC driver associated with this context. * @curve: The elliptic curve used by this transformation. * @private_key: The private key. */ struct ocs_ecc_ctx { - struct crypto_engine_ctx engine_ctx; struct ocs_ecc_dev *ecc_dev; const struct ecc_curve *curve; u64 private_key[KMB_ECC_VLI_MAX_DIGITS]; @@ -794,10 +789,6 @@ static int kmb_ecc_tctx_init(struct ocs_ecc_ctx *tctx, unsigned int curve_id) if (!tctx->curve) return -EOPNOTSUPP; - tctx->engine_ctx.op.prepare_request = NULL; - tctx->engine_ctx.op.do_one_request = kmb_ocs_ecc_do_one_request; - tctx->engine_ctx.op.unprepare_request = NULL; - return 0; } @@ -830,36 +821,38 @@ static unsigned int kmb_ocs_ecdh_max_size(struct crypto_kpp *tfm) return digits_to_bytes(tctx->curve->g.ndigits) * 2; } -static struct kpp_alg ocs_ecdh_p256 = { - .set_secret = kmb_ocs_ecdh_set_secret, - .generate_public_key = kmb_ocs_ecdh_generate_public_key, - .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, - .init = kmb_ocs_ecdh_nist_p256_init_tfm, - .exit = kmb_ocs_ecdh_exit_tfm, - .max_size = kmb_ocs_ecdh_max_size, - .base = { +static struct kpp_engine_alg ocs_ecdh_p256 = { + .base.set_secret = kmb_ocs_ecdh_set_secret, + .base.generate_public_key = kmb_ocs_ecdh_generate_public_key, + .base.compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, + .base.init = kmb_ocs_ecdh_nist_p256_init_tfm, + .base.exit = kmb_ocs_ecdh_exit_tfm, + .base.max_size = kmb_ocs_ecdh_max_size, + .base.base = { .cra_name = "ecdh-nist-p256", .cra_driver_name = "ecdh-nist-p256-keembay-ocs", .cra_priority = KMB_OCS_ECC_PRIORITY, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct ocs_ecc_ctx), }, + .op.do_one_request = kmb_ocs_ecc_do_one_request, }; -static struct kpp_alg ocs_ecdh_p384 = { - .set_secret = kmb_ocs_ecdh_set_secret, - .generate_public_key = kmb_ocs_ecdh_generate_public_key, - .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, - .init = kmb_ocs_ecdh_nist_p384_init_tfm, - .exit = kmb_ocs_ecdh_exit_tfm, - .max_size = kmb_ocs_ecdh_max_size, - .base = { +static struct kpp_engine_alg ocs_ecdh_p384 = { + .base.set_secret = kmb_ocs_ecdh_set_secret, + .base.generate_public_key = kmb_ocs_ecdh_generate_public_key, + .base.compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, + .base.init = kmb_ocs_ecdh_nist_p384_init_tfm, + .base.exit = kmb_ocs_ecdh_exit_tfm, + .base.max_size = kmb_ocs_ecdh_max_size, + .base.base = { .cra_name = "ecdh-nist-p384", .cra_driver_name = "ecdh-nist-p384-keembay-ocs", .cra_priority = KMB_OCS_ECC_PRIORITY, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct ocs_ecc_ctx), }, + .op.do_one_request = kmb_ocs_ecc_do_one_request, }; static irqreturn_t ocs_ecc_irq_handler(int irq, void *dev_id) @@ -941,14 +934,14 @@ static int kmb_ocs_ecc_probe(struct platform_device *pdev) } /* Register the KPP algo. */ - rc = crypto_register_kpp(&ocs_ecdh_p256); + rc = crypto_engine_register_kpp(&ocs_ecdh_p256); if (rc) { dev_err(dev, "Could not register OCS algorithms with Crypto API\n"); goto cleanup; } - rc = crypto_register_kpp(&ocs_ecdh_p384); + rc = crypto_engine_register_kpp(&ocs_ecdh_p384); if (rc) { dev_err(dev, "Could not register OCS algorithms with Crypto API\n"); @@ -958,7 +951,7 @@ static int kmb_ocs_ecc_probe(struct platform_device *pdev) return 0; ocs_ecdh_p384_error: - crypto_unregister_kpp(&ocs_ecdh_p256); + crypto_engine_unregister_kpp(&ocs_ecdh_p256); cleanup: crypto_engine_exit(ecc_dev->engine); @@ -977,8 +970,8 @@ static int kmb_ocs_ecc_remove(struct platform_device *pdev) ecc_dev = platform_get_drvdata(pdev); - crypto_unregister_kpp(&ocs_ecdh_p384); - crypto_unregister_kpp(&ocs_ecdh_p256); + crypto_engine_unregister_kpp(&ocs_ecdh_p384); + crypto_engine_unregister_kpp(&ocs_ecdh_p256); spin_lock(&ocs_ecc.lock); list_del(&ecc_dev->list); diff --git a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c index d4bcbed1f546..daba8ca05dbe 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c @@ -5,19 +5,21 @@ * Copyright (C) 2018-2020 Intel Corporation */ -#include <linux/completion.h> -#include <linux/delay.h> -#include <linux/dma-mapping.h> -#include <linux/interrupt.h> -#include <linux/module.h> -#include <linux/of_device.h> - #include <crypto/engine.h> +#include <crypto/hmac.h> +#include <crypto/internal/hash.h> #include <crypto/scatterwalk.h> #include <crypto/sha2.h> #include <crypto/sm3.h> -#include <crypto/hmac.h> -#include <crypto/internal/hash.h> +#include <linux/completion.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/mod_devicetable.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/string.h> #include "ocs-hcu.h" @@ -34,7 +36,6 @@ /** * struct ocs_hcu_ctx: OCS HCU Transform context. - * @engine_ctx: Crypto Engine context. * @hcu_dev: The OCS HCU device used by the transformation. * @key: The key (used only for HMAC transformations). * @key_len: The length of the key. @@ -42,7 +43,6 @@ * @is_hmac_tfm: Whether or not this is a HMAC transformation. */ struct ocs_hcu_ctx { - struct crypto_engine_ctx engine_ctx; struct ocs_hcu_dev *hcu_dev; u8 key[SHA512_BLOCK_SIZE]; size_t key_len; @@ -824,11 +824,6 @@ static void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx) { crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), sizeof(struct ocs_hcu_rctx)); - - /* Init context to 0. */ - memzero_explicit(ctx, sizeof(*ctx)); - /* Set engine ops. */ - ctx->engine_ctx.op.do_one_request = kmb_ocs_hcu_do_one_request; } static int kmb_ocs_hcu_sha_cra_init(struct crypto_tfm *tfm) @@ -883,17 +878,17 @@ static void kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm *tfm) memzero_explicit(ctx->key, sizeof(ctx->key)); } -static struct ahash_alg ocs_hcu_algs[] = { +static struct ahash_engine_alg ocs_hcu_algs[] = { #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 { - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .halg = { + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.halg = { .digestsize = SHA224_DIGEST_SIZE, .statesize = sizeof(struct ocs_hcu_rctx), .base = { @@ -907,18 +902,19 @@ static struct ahash_alg ocs_hcu_algs[] = { .cra_module = THIS_MODULE, .cra_init = kmb_ocs_hcu_sha_cra_init, } - } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, }, { - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .setkey = kmb_ocs_hcu_setkey, - .halg = { + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.setkey = kmb_ocs_hcu_setkey, + .base.halg = { .digestsize = SHA224_DIGEST_SIZE, .statesize = sizeof(struct ocs_hcu_rctx), .base = { @@ -933,18 +929,19 @@ static struct ahash_alg ocs_hcu_algs[] = { .cra_init = kmb_ocs_hcu_hmac_cra_init, .cra_exit = kmb_ocs_hcu_hmac_cra_exit, } - } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, }, #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ { - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .halg = { + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.halg = { .digestsize = SHA256_DIGEST_SIZE, .statesize = sizeof(struct ocs_hcu_rctx), .base = { @@ -958,18 +955,19 @@ static struct ahash_alg ocs_hcu_algs[] = { .cra_module = THIS_MODULE, .cra_init = kmb_ocs_hcu_sha_cra_init, } - } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, }, { - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .setkey = kmb_ocs_hcu_setkey, - .halg = { + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.setkey = kmb_ocs_hcu_setkey, + .base.halg = { .digestsize = SHA256_DIGEST_SIZE, .statesize = sizeof(struct ocs_hcu_rctx), .base = { @@ -984,17 +982,18 @@ static struct ahash_alg ocs_hcu_algs[] = { .cra_init = kmb_ocs_hcu_hmac_cra_init, .cra_exit = kmb_ocs_hcu_hmac_cra_exit, } - } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, }, { - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .halg = { + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.halg = { .digestsize = SM3_DIGEST_SIZE, .statesize = sizeof(struct ocs_hcu_rctx), .base = { @@ -1008,18 +1007,19 @@ static struct ahash_alg ocs_hcu_algs[] = { .cra_module = THIS_MODULE, .cra_init = kmb_ocs_hcu_sm3_cra_init, } - } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, }, { - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .setkey = kmb_ocs_hcu_setkey, - .halg = { + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.setkey = kmb_ocs_hcu_setkey, + .base.halg = { .digestsize = SM3_DIGEST_SIZE, .statesize = sizeof(struct ocs_hcu_rctx), .base = { @@ -1034,17 +1034,18 @@ static struct ahash_alg ocs_hcu_algs[] = { .cra_init = kmb_ocs_hcu_hmac_sm3_cra_init, .cra_exit = kmb_ocs_hcu_hmac_cra_exit, } - } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, }, { - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .halg = { + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.halg = { .digestsize = SHA384_DIGEST_SIZE, .statesize = sizeof(struct ocs_hcu_rctx), .base = { @@ -1058,18 +1059,19 @@ static struct ahash_alg ocs_hcu_algs[] = { .cra_module = THIS_MODULE, .cra_init = kmb_ocs_hcu_sha_cra_init, } - } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, }, { - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .setkey = kmb_ocs_hcu_setkey, - .halg = { + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.setkey = kmb_ocs_hcu_setkey, + .base.halg = { .digestsize = SHA384_DIGEST_SIZE, .statesize = sizeof(struct ocs_hcu_rctx), .base = { @@ -1084,17 +1086,18 @@ static struct ahash_alg ocs_hcu_algs[] = { .cra_init = kmb_ocs_hcu_hmac_cra_init, .cra_exit = kmb_ocs_hcu_hmac_cra_exit, } - } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, }, { - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .halg = { + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.halg = { .digestsize = SHA512_DIGEST_SIZE, .statesize = sizeof(struct ocs_hcu_rctx), .base = { @@ -1108,18 +1111,19 @@ static struct ahash_alg ocs_hcu_algs[] = { .cra_module = THIS_MODULE, .cra_init = kmb_ocs_hcu_sha_cra_init, } - } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, }, { - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .setkey = kmb_ocs_hcu_setkey, - .halg = { + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.setkey = kmb_ocs_hcu_setkey, + .base.halg = { .digestsize = SHA512_DIGEST_SIZE, .statesize = sizeof(struct ocs_hcu_rctx), .base = { @@ -1134,7 +1138,8 @@ static struct ahash_alg ocs_hcu_algs[] = { .cra_init = kmb_ocs_hcu_hmac_cra_init, .cra_exit = kmb_ocs_hcu_hmac_cra_exit, } - } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, }, }; @@ -1155,7 +1160,7 @@ static int kmb_ocs_hcu_remove(struct platform_device *pdev) if (!hcu_dev) return -ENODEV; - crypto_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); + crypto_engine_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); rc = crypto_engine_exit(hcu_dev->engine); @@ -1170,7 +1175,6 @@ static int kmb_ocs_hcu_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct ocs_hcu_dev *hcu_dev; - struct resource *hcu_mem; int rc; hcu_dev = devm_kzalloc(dev, sizeof(*hcu_dev), GFP_KERNEL); @@ -1184,14 +1188,7 @@ static int kmb_ocs_hcu_probe(struct platform_device *pdev) if (rc) return rc; - /* Get the memory address and remap. */ - hcu_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!hcu_mem) { - dev_err(dev, "Could not retrieve io mem resource.\n"); - return -ENODEV; - } - - hcu_dev->io_base = devm_ioremap_resource(dev, hcu_mem); + hcu_dev->io_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(hcu_dev->io_base)) return PTR_ERR(hcu_dev->io_base); @@ -1231,7 +1228,7 @@ static int kmb_ocs_hcu_probe(struct platform_device *pdev) /* Security infrastructure guarantees OCS clock is enabled. */ - rc = crypto_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); + rc = crypto_engine_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); if (rc) { dev_err(dev, "Could not register algorithms.\n"); goto cleanup; diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index e543a9e24a06..dd4464b7e00b 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -3,11 +3,13 @@ #include <linux/iopoll.h> #include <adf_accel_devices.h> #include <adf_cfg.h> +#include <adf_clock.h> #include <adf_common_drv.h> #include <adf_gen4_dc.h> #include <adf_gen4_hw_data.h> #include <adf_gen4_pfvf.h> #include <adf_gen4_pm.h> +#include <adf_gen4_timer.h> #include "adf_4xxx_hw_data.h" #include "icp_qat_hw.h" @@ -223,6 +225,8 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) ICP_ACCEL_CAPABILITIES_HKDF | ICP_ACCEL_CAPABILITIES_CHACHA_POLY | ICP_ACCEL_CAPABILITIES_AESGCM_SPC | + ICP_ACCEL_CAPABILITIES_SM3 | + ICP_ACCEL_CAPABILITIES_SM4 | ICP_ACCEL_CAPABILITIES_AES_V2; /* A set bit in fusectl1 means the feature is OFF in this SKU */ @@ -246,12 +250,19 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; } + if (fusectl1 & ICP_ACCEL_4XXX_MASK_SMX_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM3; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM4; + } + capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_SM2 | ICP_ACCEL_CAPABILITIES_ECEDMONT; if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) { capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2; capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; } @@ -317,6 +328,14 @@ static void get_admin_info(struct admin_info *admin_csrs_info) admin_csrs_info->admin_msg_lr = ADF_4XXX_ADMINMSGLR_OFFSET; } +static u32 get_heartbeat_clock(struct adf_hw_device_data *self) +{ + /* + * 4XXX uses KPT counter for HB + */ + return ADF_4XXX_KPT_COUNTER_FREQ; +} + static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR]; @@ -508,6 +527,10 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->enable_pm = adf_gen4_enable_pm; hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; hw_data->dev_config = adf_gen4_dev_config; + hw_data->start_timer = adf_gen4_timer_start; + hw_data->stop_timer = adf_gen4_timer_stop; + hw_data->get_hb_clock = get_heartbeat_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops); diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h index e5b314d2b60e..bb3d95a8fb21 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -3,6 +3,7 @@ #ifndef ADF_4XXX_HW_DATA_H_ #define ADF_4XXX_HW_DATA_H_ +#include <linux/units.h> #include <adf_accel_devices.h> /* PCIe configuration space */ @@ -64,6 +65,9 @@ #define ADF_402XX_ASYM_OBJ "qat_402xx_asym.bin" #define ADF_402XX_ADMIN_OBJ "qat_402xx_admin.bin" +/* Clocks frequency */ +#define ADF_4XXX_KPT_COUNTER_FREQ (100 * HZ_PER_MHZ) + /* qat_4xxx fuse bits are different from old GENs, redefine them */ enum icp_qat_4xxx_slice_mask { ICP_ACCEL_4XXX_MASK_CIPHER_SLICE = BIT(0), diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index 1a15600361d0..6d4e2e139ffa 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -8,6 +8,7 @@ #include <adf_cfg.h> #include <adf_common_drv.h> #include <adf_dbgfs.h> +#include <adf_heartbeat.h> #include "adf_4xxx_hw_data.h" #include "qat_compression.h" @@ -77,6 +78,8 @@ static int adf_cfg_dev_init(struct adf_accel_dev *accel_dev) if (ret) return ret; + adf_heartbeat_save_cfg_param(accel_dev, ADF_CFG_HB_TIMER_MIN_MS); + return 0; } diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 475643654e64..9c00c441b602 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -1,12 +1,14 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2014 - 2021 Intel Corporation */ #include <adf_accel_devices.h> +#include <adf_clock.h> #include <adf_common_drv.h> #include <adf_gen2_config.h> #include <adf_gen2_dc.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> #include "adf_c3xxx_hw_data.h" +#include "adf_heartbeat.h" #include "icp_qat_hw.h" /* Worker thread to service arbiter mappings */ @@ -50,6 +52,28 @@ static u32 get_ae_mask(struct adf_hw_device_data *self) return ~(fuses | straps) & ADF_C3XXX_ACCELENGINES_MASK; } +static u32 get_ts_clock(struct adf_hw_device_data *self) +{ + /* + * Timestamp update interval is 16 AE clock ticks for c3xxx. + */ + return self->clock_frequency / 16; +} + +static int measure_clock(struct adf_accel_dev *accel_dev) +{ + u32 frequency; + int ret; + + ret = adf_dev_measure_clock(accel_dev, &frequency, ADF_C3XXX_MIN_AE_FREQ, + ADF_C3XXX_MAX_AE_FREQ); + if (ret) + return ret; + + accel_dev->hw_device->clock_frequency = frequency; + return 0; +} + static u32 get_misc_bar_id(struct adf_hw_device_data *self) { return ADF_C3XXX_PMISC_BAR; @@ -127,6 +151,10 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; hw_data->disable_iov = adf_disable_sriov; hw_data->dev_config = adf_gen2_dev_config; + hw_data->measure_clock = measure_clock; + hw_data->get_hb_clock = get_ts_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->check_hb_ctrs = adf_heartbeat_check_ctrs; adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h index 336a06f11dbd..690c6a1aa172 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h @@ -3,6 +3,8 @@ #ifndef ADF_C3XXX_HW_DATA_H_ #define ADF_C3XXX_HW_DATA_H_ +#include <linux/units.h> + /* PCIe configuration space */ #define ADF_C3XXX_PMISC_BAR 0 #define ADF_C3XXX_ETR_BAR 1 @@ -19,6 +21,11 @@ #define ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS 48 #define ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS 6 +/* Clocks frequency */ +#define ADF_C3XXX_AE_FREQ (685 * HZ_PER_MHZ) +#define ADF_C3XXX_MIN_AE_FREQ (533 * HZ_PER_MHZ) +#define ADF_C3XXX_MAX_AE_FREQ (685 * HZ_PER_MHZ) + /* Firmware Binary */ #define ADF_C3XXX_FW "qat_c3xxx.bin" #define ADF_C3XXX_MMP "qat_c3xxx_mmp.bin" diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c index e14270703670..355a781693eb 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -1,12 +1,14 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2014 - 2021 Intel Corporation */ #include <adf_accel_devices.h> +#include <adf_clock.h> #include <adf_common_drv.h> #include <adf_gen2_config.h> #include <adf_gen2_dc.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> #include "adf_c62x_hw_data.h" +#include "adf_heartbeat.h" #include "icp_qat_hw.h" /* Worker thread to service arbiter mappings */ @@ -50,6 +52,28 @@ static u32 get_ae_mask(struct adf_hw_device_data *self) return ~(fuses | straps) & ADF_C62X_ACCELENGINES_MASK; } +static u32 get_ts_clock(struct adf_hw_device_data *self) +{ + /* + * Timestamp update interval is 16 AE clock ticks for c62x. + */ + return self->clock_frequency / 16; +} + +static int measure_clock(struct adf_accel_dev *accel_dev) +{ + u32 frequency; + int ret; + + ret = adf_dev_measure_clock(accel_dev, &frequency, ADF_C62X_MIN_AE_FREQ, + ADF_C62X_MAX_AE_FREQ); + if (ret) + return ret; + + accel_dev->hw_device->clock_frequency = frequency; + return 0; +} + static u32 get_misc_bar_id(struct adf_hw_device_data *self) { return ADF_C62X_PMISC_BAR; @@ -129,6 +153,10 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; hw_data->disable_iov = adf_disable_sriov; hw_data->dev_config = adf_gen2_dev_config; + hw_data->measure_clock = measure_clock; + hw_data->get_hb_clock = get_ts_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->check_hb_ctrs = adf_heartbeat_check_ctrs; adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h index 008c0a3a9769..13e6ebf6fd91 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h @@ -3,6 +3,8 @@ #ifndef ADF_C62X_HW_DATA_H_ #define ADF_C62X_HW_DATA_H_ +#include <linux/units.h> + /* PCIe configuration space */ #define ADF_C62X_SRAM_BAR 0 #define ADF_C62X_PMISC_BAR 1 @@ -19,6 +21,11 @@ #define ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS 80 #define ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS 10 +/* Clocks frequency */ +#define ADF_C62X_AE_FREQ (685 * HZ_PER_MHZ) +#define ADF_C62X_MIN_AE_FREQ (533 * HZ_PER_MHZ) +#define ADF_C62X_MAX_AE_FREQ (800 * HZ_PER_MHZ) + /* Firmware Binary */ #define ADF_C62X_FW "qat_c62x.bin" #define ADF_C62X_MMP "qat_c62x_mmp.bin" diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index 38de3aba6e8c..43622c7fca71 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -17,6 +17,8 @@ intel_qat-objs := adf_cfg.o \ adf_gen4_pm.o \ adf_gen2_dc.o \ adf_gen4_dc.o \ + adf_gen4_timer.o \ + adf_clock.o \ qat_crypto.o \ qat_compression.o \ qat_comp_algs.o \ @@ -28,6 +30,9 @@ intel_qat-objs := adf_cfg.o \ qat_bl.o intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o \ + adf_fw_counters.o \ + adf_heartbeat.o \ + adf_heartbeat_dbgfs.o \ adf_dbgfs.o intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 0399417b91fc..e57abde66f4f 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -188,6 +188,11 @@ struct adf_hw_device_data { int (*init_admin_comms)(struct adf_accel_dev *accel_dev); void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); int (*send_admin_init)(struct adf_accel_dev *accel_dev); + int (*start_timer)(struct adf_accel_dev *accel_dev); + void (*stop_timer)(struct adf_accel_dev *accel_dev); + void (*check_hb_ctrs)(struct adf_accel_dev *accel_dev); + uint32_t (*get_hb_clock)(struct adf_hw_device_data *self); + int (*measure_clock)(struct adf_accel_dev *accel_dev); int (*init_arb)(struct adf_accel_dev *accel_dev); void (*exit_arb)(struct adf_accel_dev *accel_dev); const u32 *(*get_arb_mapping)(struct adf_accel_dev *accel_dev); @@ -229,6 +234,7 @@ struct adf_hw_device_data { u8 num_accel; u8 num_logical_accel; u8 num_engines; + u32 num_hb_ctrs; }; /* CSR write macro */ @@ -241,6 +247,11 @@ struct adf_hw_device_data { #define ADF_CFG_NUM_SERVICES 4 #define ADF_SRV_TYPE_BIT_LEN 3 #define ADF_SRV_TYPE_MASK 0x7 +#define ADF_AE_ADMIN_THREAD 7 +#define ADF_NUM_THREADS_PER_AE 8 +#define ADF_NUM_PKE_STRAND 2 +#define ADF_AE_STRAND0_THREAD 8 +#define ADF_AE_STRAND1_THREAD 9 #define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev) #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) @@ -292,9 +303,12 @@ struct adf_accel_dev { unsigned long status; atomic_t ref_count; struct dentry *debugfs_dir; + struct dentry *fw_cntr_dbgfile; struct list_head list; struct module *owner; struct adf_accel_pci accel_pci_dev; + struct adf_timer *timer; + struct adf_heartbeat *heartbeat; union { struct { /* protects VF2PF interrupts access */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_admin.c b/drivers/crypto/intel/qat/qat_common/adf_admin.c index 118775ee02f2..ff790823b868 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_admin.c +++ b/drivers/crypto/intel/qat/qat_common/adf_admin.c @@ -8,6 +8,7 @@ #include <linux/dma-mapping.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" +#include "adf_heartbeat.h" #include "icp_qat_fw_init_admin.h" #define ADF_ADMIN_MAILBOX_STRIDE 0x1000 @@ -15,6 +16,7 @@ #define ADF_CONST_TABLE_SIZE 1024 #define ADF_ADMIN_POLL_DELAY_US 20 #define ADF_ADMIN_POLL_TIMEOUT_US (5 * USEC_PER_SEC) +#define ADF_ONE_AE 1 static const u8 const_tab[1024] __aligned(1024) = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -194,6 +196,22 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) return adf_send_admin(accel_dev, &req, &resp, ae_mask); } +int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp) +{ + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp; + unsigned int ae_mask = ADF_ONE_AE; + int ret; + + req.cmd_id = ICP_QAT_FW_TIMER_GET; + ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); + if (ret) + return ret; + + *timestamp = resp.timestamp; + return 0; +} + static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev, u32 *capabilities) { @@ -223,6 +241,49 @@ static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev, return 0; } +int adf_get_ae_fw_counters(struct adf_accel_dev *accel_dev, u16 ae, u64 *reqs, u64 *resps) +{ + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + int ret; + + req.cmd_id = ICP_QAT_FW_COUNTERS_GET; + + ret = adf_put_admin_msg_sync(accel_dev, ae, &req, &resp); + if (ret || resp.status) + return -EFAULT; + + *reqs = resp.req_rec_count; + *resps = resp.resp_sent_count; + + return 0; +} + +int adf_send_admin_tim_sync(struct adf_accel_dev *accel_dev, u32 cnt) +{ + u32 ae_mask = accel_dev->hw_device->ae_mask; + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp = { }; + + req.cmd_id = ICP_QAT_FW_SYNC; + req.int_timer_ticks = cnt; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + +int adf_send_admin_hb_timer(struct adf_accel_dev *accel_dev, uint32_t ticks) +{ + u32 ae_mask = accel_dev->hw_device->ae_mask; + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp; + + req.cmd_id = ICP_QAT_FW_HEARTBEAT_TIMER_SET; + req.init_cfg_ptr = accel_dev->heartbeat->dma.phy_addr; + req.heartbeat_ticks = ticks; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + /** * adf_send_admin_init() - Function sends init message to FW * @accel_dev: Pointer to acceleration device. diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h index 3ae1e5caee0e..6066dc637352 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h @@ -47,4 +47,6 @@ #define ADF_ETRMGR_CORE_AFFINITY_FORMAT \ ADF_ETRMGR_BANK "%d" ADF_ETRMGR_CORE_AFFINITY #define ADF_ACCEL_STR "Accelerator%d" +#define ADF_HEARTBEAT_TIMER "HeartbeatTimer" + #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_clock.c b/drivers/crypto/intel/qat/qat_common/adf_clock.c new file mode 100644 index 000000000000..dc0778691eb0 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_clock.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include <linux/delay.h> +#include <linux/dev_printk.h> +#include <linux/export.h> +#include <linux/math.h> +#include <linux/minmax.h> +#include <linux/time64.h> +#include <linux/types.h> +#include <linux/units.h> +#include <asm/errno.h> +#include "adf_accel_devices.h" +#include "adf_clock.h" +#include "adf_common_drv.h" + +#define MEASURE_CLOCK_RETRIES 10 +#define MEASURE_CLOCK_DELAY_US 10000 +#define ME_CLK_DIVIDER 16 +#define MEASURE_CLOCK_DELTA_THRESHOLD_US 100 + +static inline u64 timespec_to_us(const struct timespec64 *ts) +{ + return (u64)DIV_ROUND_CLOSEST_ULL(timespec64_to_ns(ts), NSEC_PER_USEC); +} + +static inline u64 timespec_to_ms(const struct timespec64 *ts) +{ + return (u64)DIV_ROUND_CLOSEST_ULL(timespec64_to_ns(ts), NSEC_PER_MSEC); +} + +u64 adf_clock_get_current_time(void) +{ + struct timespec64 ts; + + ktime_get_real_ts64(&ts); + return timespec_to_ms(&ts); +} + +static int measure_clock(struct adf_accel_dev *accel_dev, u32 *frequency) +{ + struct timespec64 ts1, ts2, ts3, ts4; + u64 timestamp1, timestamp2, temp; + u32 delta_us, tries; + int ret; + + tries = MEASURE_CLOCK_RETRIES; + do { + ktime_get_real_ts64(&ts1); + ret = adf_get_fw_timestamp(accel_dev, ×tamp1); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to get fw timestamp\n"); + return ret; + } + ktime_get_real_ts64(&ts2); + delta_us = timespec_to_us(&ts2) - timespec_to_us(&ts1); + } while (delta_us > MEASURE_CLOCK_DELTA_THRESHOLD_US && --tries); + + if (!tries) { + dev_err(&GET_DEV(accel_dev), "Excessive clock measure delay\n"); + return -ETIMEDOUT; + } + + fsleep(MEASURE_CLOCK_DELAY_US); + + tries = MEASURE_CLOCK_RETRIES; + do { + ktime_get_real_ts64(&ts3); + if (adf_get_fw_timestamp(accel_dev, ×tamp2)) { + dev_err(&GET_DEV(accel_dev), + "Failed to get fw timestamp\n"); + return -EIO; + } + ktime_get_real_ts64(&ts4); + delta_us = timespec_to_us(&ts4) - timespec_to_us(&ts3); + } while (delta_us > MEASURE_CLOCK_DELTA_THRESHOLD_US && --tries); + + if (!tries) { + dev_err(&GET_DEV(accel_dev), "Excessive clock measure delay\n"); + return -ETIMEDOUT; + } + + delta_us = timespec_to_us(&ts3) - timespec_to_us(&ts1); + temp = (timestamp2 - timestamp1) * ME_CLK_DIVIDER * 10; + temp = DIV_ROUND_CLOSEST_ULL(temp, delta_us); + /* + * Enclose the division to allow the preprocessor to precalculate it, + * and avoid promoting r-value to 64-bit before division. + */ + *frequency = temp * (HZ_PER_MHZ / 10); + + return 0; +} + +/** + * adf_dev_measure_clock() - measures device clock frequency + * @accel_dev: Pointer to acceleration device. + * @frequency: Pointer to variable where result will be stored + * @min: Minimal allowed frequency value + * @max: Maximal allowed frequency value + * + * If the measurement result will go beyond the min/max thresholds the value + * will take the value of the crossed threshold. + * + * This algorithm compares the device firmware timestamp with the kernel + * timestamp. So we can't expect too high accuracy from this measurement. + * + * Return: + * * 0 - measurement succeed + * * -ETIMEDOUT - measurement failed + */ +int adf_dev_measure_clock(struct adf_accel_dev *accel_dev, + u32 *frequency, u32 min, u32 max) +{ + int ret; + u32 freq; + + ret = measure_clock(accel_dev, &freq); + if (ret) + return ret; + + *frequency = clamp(freq, min, max); + + if (*frequency != freq) + dev_warn(&GET_DEV(accel_dev), + "Measured clock %d Hz is out of range, assuming %d\n", + freq, *frequency); + return 0; +} +EXPORT_SYMBOL_GPL(adf_dev_measure_clock); diff --git a/drivers/crypto/intel/qat/qat_common/adf_clock.h b/drivers/crypto/intel/qat/qat_common/adf_clock.h new file mode 100644 index 000000000000..e309bc0dc35c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_clock.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_CLOCK_H +#define ADF_CLOCK_H + +#include <linux/types.h> + +struct adf_accel_dev; + +int adf_dev_measure_clock(struct adf_accel_dev *accel_dev, u32 *frequency, + u32 min, u32 max); +u64 adf_clock_get_current_time(void); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h index b8132eb9bc2a..673b5044c62a 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h @@ -58,12 +58,6 @@ int adf_dev_restart(struct adf_accel_dev *accel_dev); void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data); void adf_clean_vf_map(bool); - -int adf_ctl_dev_register(void); -void adf_ctl_dev_unregister(void); -int adf_processes_dev_register(void); -void adf_processes_dev_unregister(void); - int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, struct adf_accel_dev *pf); void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, @@ -94,7 +88,11 @@ void adf_exit_aer(void); int adf_init_admin_comms(struct adf_accel_dev *accel_dev); void adf_exit_admin_comms(struct adf_accel_dev *accel_dev); int adf_send_admin_init(struct adf_accel_dev *accel_dev); +int adf_get_ae_fw_counters(struct adf_accel_dev *accel_dev, u16 ae, u64 *reqs, u64 *resps); int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay); +int adf_send_admin_tim_sync(struct adf_accel_dev *accel_dev, u32 cnt); +int adf_send_admin_hb_timer(struct adf_accel_dev *accel_dev, uint32_t ticks); +int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp); int adf_init_arb(struct adf_accel_dev *accel_dev); void adf_exit_arb(struct adf_accel_dev *accel_dev); void adf_update_ring_arb(struct adf_etr_ring_data *ring); @@ -178,8 +176,6 @@ int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned long ctx_mask, unsigned short reg_num, unsigned int regdata); -int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned short lm_addr, unsigned int value); void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned char mode); int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); @@ -193,6 +189,8 @@ int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, int adf_init_misc_wq(void); void adf_exit_misc_wq(void); bool adf_misc_wq_queue_work(struct work_struct *work); +bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, + unsigned long delay); #if defined(CONFIG_PCI_IOV) int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c index d0a2f892e6eb..04845f8d72be 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c @@ -6,6 +6,8 @@ #include "adf_cfg.h" #include "adf_common_drv.h" #include "adf_dbgfs.h" +#include "adf_fw_counters.h" +#include "adf_heartbeat_dbgfs.h" /** * adf_dbgfs_init() - add persistent debugfs entries @@ -56,6 +58,11 @@ void adf_dbgfs_add(struct adf_accel_dev *accel_dev) { if (!accel_dev->debugfs_dir) return; + + if (!accel_dev->is_vf) { + adf_fw_counters_dbgfs_add(accel_dev); + adf_heartbeat_dbgfs_add(accel_dev); + } } /** @@ -66,4 +73,9 @@ void adf_dbgfs_rm(struct adf_accel_dev *accel_dev) { if (!accel_dev->debugfs_dir) return; + + if (!accel_dev->is_vf) { + adf_heartbeat_dbgfs_rm(accel_dev); + adf_fw_counters_dbgfs_rm(accel_dev); + } } diff --git a/drivers/crypto/intel/qat/qat_common/adf_fw_counters.c b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.c new file mode 100644 index 000000000000..cb6e09ef5c9f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include <linux/bitops.h> +#include <linux/debugfs.h> +#include <linux/err.h> +#include <linux/fs.h> +#include <linux/kernel.h> +#include <linux/seq_file.h> +#include <linux/types.h> + +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_fw_counters.h" + +#define ADF_FW_COUNTERS_MAX_PADDING 16 + +enum adf_fw_counters_types { + ADF_FW_REQUESTS, + ADF_FW_RESPONSES, + ADF_FW_COUNTERS_COUNT +}; + +static const char * const adf_fw_counter_names[] = { + [ADF_FW_REQUESTS] = "Requests", + [ADF_FW_RESPONSES] = "Responses", +}; + +static_assert(ARRAY_SIZE(adf_fw_counter_names) == ADF_FW_COUNTERS_COUNT); + +struct adf_ae_counters { + u16 ae; + u64 values[ADF_FW_COUNTERS_COUNT]; +}; + +struct adf_fw_counters { + u16 ae_count; + struct adf_ae_counters ae_counters[]; +}; + +static void adf_fw_counters_parse_ae_values(struct adf_ae_counters *ae_counters, u32 ae, + u64 req_count, u64 resp_count) +{ + ae_counters->ae = ae; + ae_counters->values[ADF_FW_REQUESTS] = req_count; + ae_counters->values[ADF_FW_RESPONSES] = resp_count; +} + +static int adf_fw_counters_load_from_device(struct adf_accel_dev *accel_dev, + struct adf_fw_counters *fw_counters) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + unsigned long ae_mask; + unsigned int i; + unsigned long ae; + + /* Ignore the admin AEs */ + ae_mask = hw_data->ae_mask & ~hw_data->admin_ae_mask; + + if (hweight_long(ae_mask) > fw_counters->ae_count) + return -EINVAL; + + i = 0; + for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { + u64 req_count, resp_count; + int ret; + + ret = adf_get_ae_fw_counters(accel_dev, ae, &req_count, &resp_count); + if (ret) + return ret; + + adf_fw_counters_parse_ae_values(&fw_counters->ae_counters[i++], ae, + req_count, resp_count); + } + + return 0; +} + +static struct adf_fw_counters *adf_fw_counters_allocate(unsigned long ae_count) +{ + struct adf_fw_counters *fw_counters; + + if (unlikely(!ae_count)) + return ERR_PTR(-EINVAL); + + fw_counters = kmalloc(struct_size(fw_counters, ae_counters, ae_count), GFP_KERNEL); + if (!fw_counters) + return ERR_PTR(-ENOMEM); + + fw_counters->ae_count = ae_count; + + return fw_counters; +} + +/** + * adf_fw_counters_get() - Return FW counters for the provided device. + * @accel_dev: Pointer to a QAT acceleration device + * + * Allocates and returns a table of counters containing execution statistics + * for each non-admin AE available through the supplied acceleration device. + * The caller becomes the owner of such memory and is responsible for + * the deallocation through a call to kfree(). + * + * Returns: a pointer to a dynamically allocated struct adf_fw_counters + * on success, or a negative value on error. + */ +static struct adf_fw_counters *adf_fw_counters_get(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_fw_counters *fw_counters; + unsigned long ae_count; + int ret; + + if (!adf_dev_started(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "QAT Device not started\n"); + return ERR_PTR(-EFAULT); + } + + /* Ignore the admin AEs */ + ae_count = hweight_long(hw_data->ae_mask & ~hw_data->admin_ae_mask); + + fw_counters = adf_fw_counters_allocate(ae_count); + if (IS_ERR(fw_counters)) + return fw_counters; + + ret = adf_fw_counters_load_from_device(accel_dev, fw_counters); + if (ret) { + kfree(fw_counters); + dev_err(&GET_DEV(accel_dev), + "Failed to create QAT fw_counters file table [%d].\n", ret); + return ERR_PTR(ret); + } + + return fw_counters; +} + +static void *qat_fw_counters_seq_start(struct seq_file *sfile, loff_t *pos) +{ + struct adf_fw_counters *fw_counters = sfile->private; + + if (*pos == 0) + return SEQ_START_TOKEN; + + if (*pos > fw_counters->ae_count) + return NULL; + + return &fw_counters->ae_counters[*pos - 1]; +} + +static void *qat_fw_counters_seq_next(struct seq_file *sfile, void *v, loff_t *pos) +{ + struct adf_fw_counters *fw_counters = sfile->private; + + (*pos)++; + + if (*pos > fw_counters->ae_count) + return NULL; + + return &fw_counters->ae_counters[*pos - 1]; +} + +static void qat_fw_counters_seq_stop(struct seq_file *sfile, void *v) {} + +static int qat_fw_counters_seq_show(struct seq_file *sfile, void *v) +{ + int i; + + if (v == SEQ_START_TOKEN) { + seq_puts(sfile, "AE "); + for (i = 0; i < ADF_FW_COUNTERS_COUNT; ++i) + seq_printf(sfile, " %*s", ADF_FW_COUNTERS_MAX_PADDING, + adf_fw_counter_names[i]); + } else { + struct adf_ae_counters *ae_counters = (struct adf_ae_counters *)v; + + seq_printf(sfile, "%2d:", ae_counters->ae); + for (i = 0; i < ADF_FW_COUNTERS_COUNT; ++i) + seq_printf(sfile, " %*llu", ADF_FW_COUNTERS_MAX_PADDING, + ae_counters->values[i]); + } + seq_putc(sfile, '\n'); + + return 0; +} + +static const struct seq_operations qat_fw_counters_sops = { + .start = qat_fw_counters_seq_start, + .next = qat_fw_counters_seq_next, + .stop = qat_fw_counters_seq_stop, + .show = qat_fw_counters_seq_show, +}; + +static int qat_fw_counters_file_open(struct inode *inode, struct file *file) +{ + struct adf_accel_dev *accel_dev = inode->i_private; + struct seq_file *fw_counters_seq_file; + struct adf_fw_counters *fw_counters; + int ret; + + fw_counters = adf_fw_counters_get(accel_dev); + if (IS_ERR(fw_counters)) + return PTR_ERR(fw_counters); + + ret = seq_open(file, &qat_fw_counters_sops); + if (unlikely(ret)) { + kfree(fw_counters); + return ret; + } + + fw_counters_seq_file = file->private_data; + fw_counters_seq_file->private = fw_counters; + return ret; +} + +static int qat_fw_counters_file_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + + kfree(seq->private); + seq->private = NULL; + + return seq_release(inode, file); } + +static const struct file_operations qat_fw_counters_fops = { + .owner = THIS_MODULE, + .open = qat_fw_counters_file_open, + .read = seq_read, + .llseek = seq_lseek, + .release = qat_fw_counters_file_release, +}; + +/** + * adf_fw_counters_dbgfs_add() - Create a debugfs file containing FW + * execution counters. + * @accel_dev: Pointer to a QAT acceleration device + * + * Function creates a file to display a table with statistics for the given + * QAT acceleration device. The table stores device specific execution values + * for each AE, such as the number of requests sent to the FW and responses + * received from the FW. + * + * Return: void + */ +void adf_fw_counters_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + accel_dev->fw_cntr_dbgfile = debugfs_create_file("fw_counters", 0400, + accel_dev->debugfs_dir, + accel_dev, + &qat_fw_counters_fops); +} + +/** + * adf_fw_counters_dbgfs_rm() - Remove the debugfs file containing FW counters. + * @accel_dev: Pointer to a QAT acceleration device. + * + * Function removes the file providing the table of statistics for the given + * QAT acceleration device. + * + * Return: void + */ +void adf_fw_counters_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + debugfs_remove(accel_dev->fw_cntr_dbgfile); + accel_dev->fw_cntr_dbgfile = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_fw_counters.h b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.h new file mode 100644 index 000000000000..91b3b6a95f1f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_FW_COUNTERS_H +#define ADF_FW_COUNTERS_H + +struct adf_accel_dev; + +void adf_fw_counters_dbgfs_add(struct adf_accel_dev *accel_dev); +void adf_fw_counters_dbgfs_rm(struct adf_accel_dev *accel_dev); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c index eeb30da7587a..c27ff6d18e11 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c @@ -7,6 +7,7 @@ #include "adf_common_drv.h" #include "qat_crypto.h" #include "qat_compression.h" +#include "adf_heartbeat.h" #include "adf_transport_access_macros.h" static int adf_gen2_crypto_dev_config(struct adf_accel_dev *accel_dev) @@ -195,6 +196,12 @@ int adf_gen2_dev_config(struct adf_accel_dev *accel_dev) if (ret) goto err; + ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); + if (ret) + goto err; + + adf_heartbeat_save_cfg_param(accel_dev, ADF_CFG_HB_TIMER_DEFAULT_MS); + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); return ret; diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h index e4bc07529be4..6bd341061de4 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h @@ -145,6 +145,9 @@ do { \ #define ADF_GEN2_CERRSSMSH(i) ((i) * 0x4000 + 0x10) #define ADF_GEN2_ERRSSMSH_EN BIT(3) +/* Number of heartbeat counter pairs */ +#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE + /* Interrupts */ #define ADF_GEN2_SMIAPF0_MASK_OFFSET (0x3A000 + 0x28) #define ADF_GEN2_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h index 4fb4b3df5a18..02d7a019ebf8 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h @@ -136,6 +136,9 @@ do { \ #define ADF_GEN4_VFLNOTIFY BIT(7) +/* Number of heartbeat counter pairs */ +#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE + void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h index dd112923e006..c2768762cca3 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h @@ -35,7 +35,7 @@ #define ADF_GEN4_PM_MSG_PENDING BIT(0) #define ADF_GEN4_PM_MSG_PAYLOAD_BIT_MASK GENMASK(28, 1) -#define ADF_GEN4_PM_DEFAULT_IDLE_FILTER (0x0) +#define ADF_GEN4_PM_DEFAULT_IDLE_FILTER (0x6) #define ADF_GEN4_PM_MAX_IDLE_FILTER (0x7) #define ADF_GEN4_PM_DEFAULT_IDLE_SUPPORT (0x1) diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c new file mode 100644 index 000000000000..646c57922fcd --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include <linux/container_of.h> +#include <linux/dev_printk.h> +#include <linux/export.h> +#include <linux/jiffies.h> +#include <linux/ktime.h> +#include <linux/slab.h> +#include <linux/workqueue.h> + +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_gen4_timer.h" + +#define ADF_GEN4_TIMER_PERIOD_MS 200 + +/* This periodic update is used to trigger HB, RL & TL fw events */ +static void work_handler(struct work_struct *work) +{ + struct adf_accel_dev *accel_dev; + struct adf_timer *timer_ctx; + u32 time_periods; + + timer_ctx = container_of(to_delayed_work(work), struct adf_timer, work_ctx); + accel_dev = timer_ctx->accel_dev; + + adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, + msecs_to_jiffies(ADF_GEN4_TIMER_PERIOD_MS)); + + time_periods = div_u64(ktime_ms_delta(ktime_get_real(), timer_ctx->initial_ktime), + ADF_GEN4_TIMER_PERIOD_MS); + + if (adf_send_admin_tim_sync(accel_dev, time_periods)) + dev_err(&GET_DEV(accel_dev), "Failed to synchronize qat timer\n"); +} + +int adf_gen4_timer_start(struct adf_accel_dev *accel_dev) +{ + struct adf_timer *timer_ctx; + + timer_ctx = kzalloc(sizeof(*timer_ctx), GFP_KERNEL); + if (!timer_ctx) + return -ENOMEM; + + timer_ctx->accel_dev = accel_dev; + accel_dev->timer = timer_ctx; + timer_ctx->initial_ktime = ktime_get_real(); + + INIT_DELAYED_WORK(&timer_ctx->work_ctx, work_handler); + adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, + msecs_to_jiffies(ADF_GEN4_TIMER_PERIOD_MS)); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_timer_start); + +void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev) +{ + struct adf_timer *timer_ctx = accel_dev->timer; + + if (!timer_ctx) + return; + + cancel_delayed_work_sync(&timer_ctx->work_ctx); + + kfree(timer_ctx); + accel_dev->timer = NULL; +} +EXPORT_SYMBOL_GPL(adf_gen4_timer_stop); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h new file mode 100644 index 000000000000..66a709e7b358 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_GEN4_TIMER_H_ +#define ADF_GEN4_TIMER_H_ + +#include <linux/ktime.h> +#include <linux/workqueue.h> + +struct adf_accel_dev; + +struct adf_timer { + struct adf_accel_dev *accel_dev; + struct delayed_work work_ctx; + ktime_t initial_ktime; +}; + +int adf_gen4_timer_start(struct adf_accel_dev *accel_dev); +void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev); + +#endif /* ADF_GEN4_TIMER_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat.c b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.c new file mode 100644 index 000000000000..beef9a5f6c75 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.c @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include <linux/dev_printk.h> +#include <linux/dma-mapping.h> +#include <linux/export.h> +#include <linux/kernel.h> +#include <linux/kstrtox.h> +#include <linux/overflow.h> +#include <linux/string.h> +#include <linux/slab.h> +#include <linux/types.h> +#include <asm/errno.h> +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "adf_clock.h" +#include "adf_common_drv.h" +#include "adf_heartbeat.h" +#include "adf_transport_internal.h" +#include "icp_qat_fw_init_admin.h" + +#define ADF_HB_EMPTY_SIG 0xA5A5A5A5 + +/* Heartbeat counter pair */ +struct hb_cnt_pair { + __u16 resp_heartbeat_cnt; + __u16 req_heartbeat_cnt; +}; + +static int adf_hb_check_polling_freq(struct adf_accel_dev *accel_dev) +{ + u64 curr_time = adf_clock_get_current_time(); + u64 polling_time = curr_time - accel_dev->heartbeat->last_hb_check_time; + + if (polling_time < accel_dev->heartbeat->hb_timer) { + dev_warn(&GET_DEV(accel_dev), + "HB polling too frequent. Configured HB timer %d ms\n", + accel_dev->heartbeat->hb_timer); + return -EINVAL; + } + + accel_dev->heartbeat->last_hb_check_time = curr_time; + return 0; +} + +/** + * validate_hb_ctrs_cnt() - checks if the number of heartbeat counters should + * be updated by one to support the currently loaded firmware. + * @accel_dev: Pointer to acceleration device. + * + * Return: + * * true - hb_ctrs must increased by ADF_NUM_PKE_STRAND + * * false - no changes needed + */ +static bool validate_hb_ctrs_cnt(struct adf_accel_dev *accel_dev) +{ + const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs; + const size_t max_aes = accel_dev->hw_device->num_engines; + const size_t hb_struct_size = sizeof(struct hb_cnt_pair); + const size_t exp_diff_size = array3_size(ADF_NUM_PKE_STRAND, max_aes, + hb_struct_size); + const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); + const size_t stats_size = size_mul(dev_ctrs, hb_struct_size); + const u32 exp_diff_cnt = exp_diff_size / sizeof(u32); + const u32 stats_el_cnt = stats_size / sizeof(u32); + struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr; + const u32 *mem_to_chk = (u32 *)(hb_stats + dev_ctrs); + u32 el_diff_cnt = 0; + int i; + + /* count how many bytes are different from pattern */ + for (i = 0; i < stats_el_cnt; i++) { + if (mem_to_chk[i] == ADF_HB_EMPTY_SIG) + break; + + el_diff_cnt++; + } + + return el_diff_cnt && el_diff_cnt == exp_diff_cnt; +} + +void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev) +{ + struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr; + const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs; + const size_t max_aes = accel_dev->hw_device->num_engines; + const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); + const size_t stats_size = size_mul(dev_ctrs, sizeof(struct hb_cnt_pair)); + const size_t mem_items_to_fill = size_mul(stats_size, 2) / sizeof(u32); + + /* fill hb stats memory with pattern */ + memset32((uint32_t *)hb_stats, ADF_HB_EMPTY_SIG, mem_items_to_fill); + accel_dev->heartbeat->ctrs_cnt_checked = false; +} +EXPORT_SYMBOL_GPL(adf_heartbeat_check_ctrs); + +static int get_timer_ticks(struct adf_accel_dev *accel_dev, unsigned int *value) +{ + char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; + u32 timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS; + int cfg_read_status; + u32 ticks; + int ret; + + cfg_read_status = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_HEARTBEAT_TIMER, timer_str); + if (cfg_read_status == 0) { + if (kstrtouint(timer_str, 10, &timer_ms)) + dev_dbg(&GET_DEV(accel_dev), + "kstrtouint failed to parse the %s, param value", + ADF_HEARTBEAT_TIMER); + } + + if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) { + dev_err(&GET_DEV(accel_dev), "Timer cannot be less than %u\n", + ADF_CFG_HB_TIMER_MIN_MS); + return -EINVAL; + } + + /* + * On 4xxx devices adf_timer is responsible for HB updates and + * its period is fixed to 200ms + */ + if (accel_dev->timer) + timer_ms = ADF_CFG_HB_TIMER_MIN_MS; + + ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks); + if (ret) + return ret; + + adf_heartbeat_save_cfg_param(accel_dev, timer_ms); + + accel_dev->heartbeat->hb_timer = timer_ms; + *value = ticks; + + return 0; +} + +static int check_ae(struct hb_cnt_pair *curr, struct hb_cnt_pair *prev, + u16 *count, const size_t hb_ctrs) +{ + size_t thr; + + /* loop through all threads in AE */ + for (thr = 0; thr < hb_ctrs; thr++) { + u16 req = curr[thr].req_heartbeat_cnt; + u16 resp = curr[thr].resp_heartbeat_cnt; + u16 last = prev[thr].resp_heartbeat_cnt; + + if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && resp == last) { + u16 retry = ++count[thr]; + + if (retry >= ADF_CFG_HB_COUNT_THRESHOLD) + return -EIO; + + } else { + count[thr] = 0; + } + } + return 0; +} + +static int adf_hb_get_status(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct hb_cnt_pair *live_stats, *last_stats, *curr_stats; + const size_t hb_ctrs = hw_device->num_hb_ctrs; + const unsigned long ae_mask = hw_device->ae_mask; + const size_t max_aes = hw_device->num_engines; + const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); + const size_t stats_size = size_mul(dev_ctrs, sizeof(*curr_stats)); + struct hb_cnt_pair *ae_curr_p, *ae_prev_p; + u16 *count_fails, *ae_count_p; + size_t ae_offset; + size_t ae = 0; + int ret = 0; + + if (!accel_dev->heartbeat->ctrs_cnt_checked) { + if (validate_hb_ctrs_cnt(accel_dev)) + hw_device->num_hb_ctrs += ADF_NUM_PKE_STRAND; + + accel_dev->heartbeat->ctrs_cnt_checked = true; + } + + live_stats = accel_dev->heartbeat->dma.virt_addr; + last_stats = live_stats + dev_ctrs; + count_fails = (u16 *)(last_stats + dev_ctrs); + + curr_stats = kmemdup(live_stats, stats_size, GFP_KERNEL); + if (!curr_stats) + return -ENOMEM; + + /* loop through active AEs */ + for_each_set_bit(ae, &ae_mask, max_aes) { + ae_offset = size_mul(ae, hb_ctrs); + ae_curr_p = curr_stats + ae_offset; + ae_prev_p = last_stats + ae_offset; + ae_count_p = count_fails + ae_offset; + + ret = check_ae(ae_curr_p, ae_prev_p, ae_count_p, hb_ctrs); + if (ret) + break; + } + + /* Copy current stats for the next iteration */ + memcpy(last_stats, curr_stats, stats_size); + kfree(curr_stats); + + return ret; +} + +void adf_heartbeat_status(struct adf_accel_dev *accel_dev, + enum adf_device_heartbeat_status *hb_status) +{ + struct adf_heartbeat *hb; + + if (!adf_dev_started(accel_dev) || + test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) { + *hb_status = HB_DEV_UNRESPONSIVE; + return; + } + + if (adf_hb_check_polling_freq(accel_dev) == -EINVAL) { + *hb_status = HB_DEV_UNSUPPORTED; + return; + } + + hb = accel_dev->heartbeat; + hb->hb_sent_counter++; + + if (adf_hb_get_status(accel_dev)) { + dev_err(&GET_DEV(accel_dev), + "Heartbeat ERROR: QAT is not responding.\n"); + *hb_status = HB_DEV_UNRESPONSIVE; + hb->hb_failed_counter++; + return; + } + + *hb_status = HB_DEV_ALIVE; +} + +int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms, + u32 *value) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 clk_per_sec; + + /* HB clock may be different than AE clock */ + if (!hw_data->get_hb_clock) + return -EINVAL; + + clk_per_sec = hw_data->get_hb_clock(hw_data); + *value = time_ms * (clk_per_sec / MSEC_PER_SEC); + + return 0; +} + +int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, + unsigned int timer_ms) +{ + char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + + snprintf(timer_str, sizeof(timer_str), "%u", timer_ms); + return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_HEARTBEAT_TIMER, timer_str, + ADF_STR); +} +EXPORT_SYMBOL_GPL(adf_heartbeat_save_cfg_param); + +int adf_heartbeat_init(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb; + + hb = kzalloc(sizeof(*hb), GFP_KERNEL); + if (!hb) + goto err_ret; + + hb->dma.virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + &hb->dma.phy_addr, GFP_KERNEL); + if (!hb->dma.virt_addr) + goto err_free; + + /* + * Default set this flag as true to avoid unnecessary checks, + * it will be reset on platforms that need such a check + */ + hb->ctrs_cnt_checked = true; + accel_dev->heartbeat = hb; + + return 0; + +err_free: + kfree(hb); +err_ret: + return -ENOMEM; +} + +int adf_heartbeat_start(struct adf_accel_dev *accel_dev) +{ + unsigned int timer_ticks; + int ret; + + if (!accel_dev->heartbeat) { + dev_warn(&GET_DEV(accel_dev), "Heartbeat instance not found!"); + return -EFAULT; + } + + if (accel_dev->hw_device->check_hb_ctrs) + accel_dev->hw_device->check_hb_ctrs(accel_dev); + + ret = get_timer_ticks(accel_dev, &timer_ticks); + if (ret) + return ret; + + ret = adf_send_admin_hb_timer(accel_dev, timer_ticks); + if (ret) + dev_warn(&GET_DEV(accel_dev), "Heartbeat not supported!"); + + return ret; +} + +void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb = accel_dev->heartbeat; + + if (!hb) + return; + + if (hb->dma.virt_addr) + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + hb->dma.virt_addr, hb->dma.phy_addr); + + kfree(hb); + accel_dev->heartbeat = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat.h b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.h new file mode 100644 index 000000000000..b22e3cb29798 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_HEARTBEAT_H_ +#define ADF_HEARTBEAT_H_ + +#include <linux/types.h> + +struct adf_accel_dev; +struct dentry; + +#define ADF_CFG_HB_TIMER_MIN_MS 200 +#define ADF_CFG_HB_TIMER_DEFAULT_MS 500 +#define ADF_CFG_HB_COUNT_THRESHOLD 3 + +enum adf_device_heartbeat_status { + HB_DEV_UNRESPONSIVE = 0, + HB_DEV_ALIVE, + HB_DEV_UNSUPPORTED, +}; + +struct adf_heartbeat { + unsigned int hb_sent_counter; + unsigned int hb_failed_counter; + unsigned int hb_timer; + u64 last_hb_check_time; + bool ctrs_cnt_checked; + struct hb_dma_addr { + dma_addr_t phy_addr; + void *virt_addr; + } dma; + struct { + struct dentry *base_dir; + struct dentry *status; + struct dentry *cfg; + struct dentry *sent; + struct dentry *failed; + } dbgfs; +}; + +#ifdef CONFIG_DEBUG_FS +int adf_heartbeat_init(struct adf_accel_dev *accel_dev); +int adf_heartbeat_start(struct adf_accel_dev *accel_dev); +void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev); + +int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms, + uint32_t *value); +int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, + unsigned int timer_ms); +void adf_heartbeat_status(struct adf_accel_dev *accel_dev, + enum adf_device_heartbeat_status *hb_status); +void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev); + +#else +static inline int adf_heartbeat_init(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline int adf_heartbeat_start(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev) +{ +} + +static inline int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, + unsigned int timer_ms) +{ + return 0; +} + +static inline void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev) +{ +} +#endif +#endif /* ADF_HEARTBEAT_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c new file mode 100644 index 000000000000..803cbfd838f0 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include <linux/debugfs.h> +#include <linux/errno.h> +#include <linux/export.h> +#include <linux/fs.h> +#include <linux/kernel.h> +#include <linux/kstrtox.h> +#include <linux/types.h> +#include "adf_cfg.h" +#include "adf_common_drv.h" +#include "adf_heartbeat.h" +#include "adf_heartbeat_dbgfs.h" + +#define HB_OK 0 +#define HB_ERROR -1 +#define HB_STATUS_MAX_STRLEN 4 +#define HB_STATS_MAX_STRLEN 16 + +static ssize_t adf_hb_stats_read(struct file *file, char __user *user_buffer, + size_t count, loff_t *ppos) +{ + char buf[HB_STATS_MAX_STRLEN]; + unsigned int *value; + int len; + + if (*ppos > 0) + return 0; + + value = file->private_data; + len = scnprintf(buf, sizeof(buf), "%u\n", *value); + + return simple_read_from_buffer(user_buffer, count, ppos, buf, len + 1); +} + +static const struct file_operations adf_hb_stats_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = adf_hb_stats_read, +}; + +static ssize_t adf_hb_status_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + enum adf_device_heartbeat_status hb_status; + char ret_str[HB_STATUS_MAX_STRLEN]; + struct adf_accel_dev *accel_dev; + int ret_code; + size_t len; + + if (*ppos > 0) + return 0; + + accel_dev = file->private_data; + ret_code = HB_OK; + + adf_heartbeat_status(accel_dev, &hb_status); + + if (hb_status != HB_DEV_ALIVE) + ret_code = HB_ERROR; + + len = scnprintf(ret_str, sizeof(ret_str), "%d\n", ret_code); + + return simple_read_from_buffer(user_buf, count, ppos, ret_str, len + 1); +} + +static const struct file_operations adf_hb_status_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = adf_hb_status_read, +}; + +static ssize_t adf_hb_cfg_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + struct adf_accel_dev *accel_dev; + unsigned int timer_ms; + int len; + + if (*ppos > 0) + return 0; + + accel_dev = file->private_data; + timer_ms = accel_dev->heartbeat->hb_timer; + len = scnprintf(timer_str, sizeof(timer_str), "%u\n", timer_ms); + + return simple_read_from_buffer(user_buf, count, ppos, timer_str, + len + 1); +} + +static ssize_t adf_hb_cfg_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + char input_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; + struct adf_accel_dev *accel_dev; + int ret, written_chars; + unsigned int timer_ms; + u32 ticks; + + accel_dev = file->private_data; + timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS; + + /* last byte left as string termination */ + if (count > sizeof(input_str) - 1) + return -EINVAL; + + written_chars = simple_write_to_buffer(input_str, sizeof(input_str) - 1, + ppos, user_buf, count); + if (written_chars > 0) { + ret = kstrtouint(input_str, 10, &timer_ms); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "heartbeat_cfg: Invalid value\n"); + return ret; + } + + if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) { + dev_err(&GET_DEV(accel_dev), + "heartbeat_cfg: Invalid value\n"); + return -EINVAL; + } + + /* + * On 4xxx devices adf_timer is responsible for HB updates and + * its period is fixed to 200ms + */ + if (accel_dev->timer) + timer_ms = ADF_CFG_HB_TIMER_MIN_MS; + + ret = adf_heartbeat_save_cfg_param(accel_dev, timer_ms); + if (ret) + return ret; + + ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks); + if (ret) + return ret; + + ret = adf_send_admin_hb_timer(accel_dev, ticks); + if (ret) + return ret; + + accel_dev->heartbeat->hb_timer = timer_ms; + } + + return written_chars; +} + +static const struct file_operations adf_hb_cfg_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = adf_hb_cfg_read, + .write = adf_hb_cfg_write, +}; + +void adf_heartbeat_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb = accel_dev->heartbeat; + + if (!hb) + return; + + hb->dbgfs.base_dir = debugfs_create_dir("heartbeat", accel_dev->debugfs_dir); + hb->dbgfs.status = debugfs_create_file("status", 0400, hb->dbgfs.base_dir, + accel_dev, &adf_hb_status_fops); + hb->dbgfs.sent = debugfs_create_file("queries_sent", 0400, hb->dbgfs.base_dir, + &hb->hb_sent_counter, &adf_hb_stats_fops); + hb->dbgfs.failed = debugfs_create_file("queries_failed", 0400, hb->dbgfs.base_dir, + &hb->hb_failed_counter, &adf_hb_stats_fops); + hb->dbgfs.cfg = debugfs_create_file("config", 0600, hb->dbgfs.base_dir, + accel_dev, &adf_hb_cfg_fops); +} +EXPORT_SYMBOL_GPL(adf_heartbeat_dbgfs_add); + +void adf_heartbeat_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb = accel_dev->heartbeat; + + if (!hb) + return; + + debugfs_remove(hb->dbgfs.status); + hb->dbgfs.status = NULL; + debugfs_remove(hb->dbgfs.sent); + hb->dbgfs.sent = NULL; + debugfs_remove(hb->dbgfs.failed); + hb->dbgfs.failed = NULL; + debugfs_remove(hb->dbgfs.cfg); + hb->dbgfs.cfg = NULL; + debugfs_remove(hb->dbgfs.base_dir); + hb->dbgfs.base_dir = NULL; +} +EXPORT_SYMBOL_GPL(adf_heartbeat_dbgfs_rm); diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.h b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.h new file mode 100644 index 000000000000..84dd29ea6454 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_HEARTBEAT_DBGFS_H_ +#define ADF_HEARTBEAT_DBGFS_H_ + +struct adf_accel_dev; + +void adf_heartbeat_dbgfs_add(struct adf_accel_dev *accel_dev); +void adf_heartbeat_dbgfs_rm(struct adf_accel_dev *accel_dev); + +#endif /* ADF_HEARTBEAT_DBGFS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_init.c b/drivers/crypto/intel/qat/qat_common/adf_init.c index 826179c98524..89001fe92e76 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_init.c +++ b/drivers/crypto/intel/qat/qat_common/adf_init.c @@ -8,6 +8,7 @@ #include "adf_cfg.h" #include "adf_common_drv.h" #include "adf_dbgfs.h" +#include "adf_heartbeat.h" static LIST_HEAD(service_table); static DEFINE_MUTEX(service_lock); @@ -129,6 +130,8 @@ static int adf_dev_init(struct adf_accel_dev *accel_dev) return -EFAULT; } + adf_heartbeat_init(accel_dev); + /* * Subservice initialisation is divided into two stages: init and start. * This is to facilitate any ordering dependencies between services @@ -163,6 +166,7 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct service_hndl *service; struct list_head *list_itr; + int ret; set_bit(ADF_STATUS_STARTING, &accel_dev->status); @@ -177,6 +181,14 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) return -EFAULT; } + if (hw_data->measure_clock) { + ret = hw_data->measure_clock(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed measure device clock\n"); + return ret; + } + } + /* Set ssm watch dog timer */ if (hw_data->set_ssm_wdtimer) hw_data->set_ssm_wdtimer(accel_dev); @@ -187,6 +199,16 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) return -EFAULT; } + if (hw_data->start_timer) { + ret = hw_data->start_timer(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to start internal sync timer\n"); + return ret; + } + } + + adf_heartbeat_start(accel_dev); + list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); if (service->event_hld(accel_dev, ADF_EVENT_START)) { @@ -235,6 +257,7 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) */ static void adf_dev_stop(struct adf_accel_dev *accel_dev) { + struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct service_hndl *service; struct list_head *list_itr; bool wait = false; @@ -270,6 +293,9 @@ static void adf_dev_stop(struct adf_accel_dev *accel_dev) } } + if (hw_data->stop_timer) + hw_data->stop_timer(accel_dev); + if (wait) msleep(100); @@ -326,6 +352,8 @@ static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) clear_bit(accel_dev->accel_id, service->init_status); } + adf_heartbeat_shutdown(accel_dev); + hw_data->disable_iov(accel_dev); if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_isr.c index ad9e135b8560..2aba194a7c29 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_isr.c @@ -380,3 +380,9 @@ bool adf_misc_wq_queue_work(struct work_struct *work) { return queue_work(adf_misc_wq, work); } + +bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, + unsigned long delay) +{ + return queue_delayed_work(adf_misc_wq, work, delay); +} diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h index 56cb827f93ea..3e968a4bcc9c 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h @@ -16,6 +16,8 @@ enum icp_qat_fw_init_admin_cmd_id { ICP_QAT_FW_HEARTBEAT_SYNC = 7, ICP_QAT_FW_HEARTBEAT_GET = 8, ICP_QAT_FW_COMP_CAPABILITY_GET = 9, + ICP_QAT_FW_HEARTBEAT_TIMER_SET = 13, + ICP_QAT_FW_TIMER_GET = 19, ICP_QAT_FW_PM_STATE_CONFIG = 128, }; @@ -37,6 +39,12 @@ struct icp_qat_fw_init_admin_req { __u16 ibuf_size_in_kb; __u16 resrvd3; }; + struct { + __u32 int_timer_ticks; + }; + struct { + __u32 heartbeat_ticks; + }; __u32 idle_filter; }; @@ -97,19 +105,6 @@ struct icp_qat_fw_init_admin_resp { }; } __packed; -#define ICP_QAT_FW_COMN_HEARTBEAT_OK 0 -#define ICP_QAT_FW_COMN_HEARTBEAT_BLOCKED 1 -#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS 0 -#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_MASK 0x1 -#define ICP_QAT_FW_COMN_STATUS_RESRVD_FLD_MASK 0xFE -#define ICP_QAT_FW_COMN_HEARTBEAT_HDR_FLAG_GET(hdr_t) \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_GET(hdr_t.flags) - -#define ICP_QAT_FW_COMN_HEARTBEAT_HDR_FLAG_SET(hdr_t, val) \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_SET(hdr_t, val) +#define ICP_QAT_FW_SYNC ICP_QAT_FW_HEARTBEAT_SYNC -#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_GET(flags) \ - QAT_FIELD_GET(flags, \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS, \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_MASK) #endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h index a65059e56248..0c8883e2ccc6 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h @@ -97,7 +97,10 @@ enum icp_qat_capabilities_mask { ICP_ACCEL_CAPABILITIES_SHA3_EXT = BIT(15), ICP_ACCEL_CAPABILITIES_AESGCM_SPC = BIT(16), ICP_ACCEL_CAPABILITIES_CHACHA_POLY = BIT(17), - /* Bits 18-21 are currently reserved */ + ICP_ACCEL_CAPABILITIES_SM2 = BIT(18), + ICP_ACCEL_CAPABILITIES_SM3 = BIT(19), + ICP_ACCEL_CAPABILITIES_SM4 = BIT(20), + /* Bit 21 is currently reserved */ ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY = BIT(22), ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64 = BIT(23), ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION = BIT(24), diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c index 3f1f35283266..7842a9f22178 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.c +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c @@ -234,8 +234,7 @@ static void qat_free_dc_data(struct adf_accel_dev *accel_dev) dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz, DMA_FROM_DEVICE); - memset(dc_data->ovf_buff, 0, dc_data->ovf_buff_sz); - kfree(dc_data->ovf_buff); + kfree_sensitive(dc_data->ovf_buff); devm_kfree(dev, dc_data); accel_dev->dc_data = NULL; } diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index ce837bcc1cab..4bd150d1441a 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -11,7 +11,7 @@ #include "icp_qat_hal.h" #include "icp_qat_fw_loader_handle.h" -#define UWORD_CPYBUF_SIZE 1024 +#define UWORD_CPYBUF_SIZE 1024U #define INVLD_UWORD 0xffffffffffull #define PID_MINOR_REV 0xf #define PID_MAJOR_REV (0xf << 4) @@ -1986,10 +1986,7 @@ static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, uw_relative_addr = 0; words_num = encap_page->micro_words_num; while (words_num) { - if (words_num < UWORD_CPYBUF_SIZE) - cpylen = words_num; - else - cpylen = UWORD_CPYBUF_SIZE; + cpylen = min(words_num, UWORD_CPYBUF_SIZE); /* load the buffer */ for (i = 0; i < cpylen; i++) diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 1ebe0b351fae..09551f949126 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -7,6 +7,7 @@ #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> #include "adf_dh895xcc_hw_data.h" +#include "adf_heartbeat.h" #include "icp_qat_hw.h" #define ADF_DH895XCC_VF_MSK 0xFFFFFFFF @@ -44,6 +45,14 @@ static u32 get_misc_bar_id(struct adf_hw_device_data *self) return ADF_DH895XCC_PMISC_BAR; } +static u32 get_ts_clock(struct adf_hw_device_data *self) +{ + /* + * Timestamp update interval is 16 AE clock ticks for dh895xcc. + */ + return self->clock_frequency / 16; +} + static u32 get_etr_bar_id(struct adf_hw_device_data *self) { return ADF_DH895XCC_ETR_BAR; @@ -237,6 +246,10 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->reset_device = adf_reset_sbr; hw_data->disable_iov = adf_disable_sriov; hw_data->dev_config = adf_gen2_dev_config; + hw_data->clock_frequency = ADF_DH895X_AE_FREQ; + hw_data->get_hb_clock = get_ts_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->check_hb_ctrs = adf_heartbeat_check_ctrs; adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); hw_data->pfvf_ops.enable_vf2pf_interrupts = enable_vf2pf_interrupts; diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index 7b674bbe4192..cd3a21985455 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -3,6 +3,8 @@ #ifndef ADF_DH895x_HW_DATA_H_ #define ADF_DH895x_HW_DATA_H_ +#include <linux/units.h> + /* PCIe configuration space */ #define ADF_DH895XCC_SRAM_BAR 0 #define ADF_DH895XCC_PMISC_BAR 1 @@ -30,6 +32,9 @@ #define ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS 96 #define ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS 12 +/* Clocks frequency */ +#define ADF_DH895X_AE_FREQ (933 * HZ_PER_MHZ) + /* FW names */ #define ADF_DH895XCC_FW "qat_895xcc.bin" #define ADF_DH895XCC_MMP "qat_895xcc_mmp.bin" |