diff options
Diffstat (limited to 'drivers/crypto/intel/qat/qat_common')
26 files changed, 1296 insertions, 31 deletions
diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index 38de3aba6e8c..43622c7fca71 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -17,6 +17,8 @@ intel_qat-objs := adf_cfg.o \ adf_gen4_pm.o \ adf_gen2_dc.o \ adf_gen4_dc.o \ + adf_gen4_timer.o \ + adf_clock.o \ qat_crypto.o \ qat_compression.o \ qat_comp_algs.o \ @@ -28,6 +30,9 @@ intel_qat-objs := adf_cfg.o \ qat_bl.o intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o \ + adf_fw_counters.o \ + adf_heartbeat.o \ + adf_heartbeat_dbgfs.o \ adf_dbgfs.o intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 0399417b91fc..e57abde66f4f 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -188,6 +188,11 @@ struct adf_hw_device_data { int (*init_admin_comms)(struct adf_accel_dev *accel_dev); void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); int (*send_admin_init)(struct adf_accel_dev *accel_dev); + int (*start_timer)(struct adf_accel_dev *accel_dev); + void (*stop_timer)(struct adf_accel_dev *accel_dev); + void (*check_hb_ctrs)(struct adf_accel_dev *accel_dev); + uint32_t (*get_hb_clock)(struct adf_hw_device_data *self); + int (*measure_clock)(struct adf_accel_dev *accel_dev); int (*init_arb)(struct adf_accel_dev *accel_dev); void (*exit_arb)(struct adf_accel_dev *accel_dev); const u32 *(*get_arb_mapping)(struct adf_accel_dev *accel_dev); @@ -229,6 +234,7 @@ struct adf_hw_device_data { u8 num_accel; u8 num_logical_accel; u8 num_engines; + u32 num_hb_ctrs; }; /* CSR write macro */ @@ -241,6 +247,11 @@ struct adf_hw_device_data { #define ADF_CFG_NUM_SERVICES 4 #define ADF_SRV_TYPE_BIT_LEN 3 #define ADF_SRV_TYPE_MASK 0x7 +#define ADF_AE_ADMIN_THREAD 7 +#define ADF_NUM_THREADS_PER_AE 8 +#define ADF_NUM_PKE_STRAND 2 +#define ADF_AE_STRAND0_THREAD 8 +#define ADF_AE_STRAND1_THREAD 9 #define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev) #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) @@ -292,9 +303,12 @@ struct adf_accel_dev { unsigned long status; atomic_t ref_count; struct dentry *debugfs_dir; + struct dentry *fw_cntr_dbgfile; struct list_head list; struct module *owner; struct adf_accel_pci accel_pci_dev; + struct adf_timer *timer; + struct adf_heartbeat *heartbeat; union { struct { /* protects VF2PF interrupts access */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_admin.c b/drivers/crypto/intel/qat/qat_common/adf_admin.c index 118775ee02f2..ff790823b868 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_admin.c +++ b/drivers/crypto/intel/qat/qat_common/adf_admin.c @@ -8,6 +8,7 @@ #include <linux/dma-mapping.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" +#include "adf_heartbeat.h" #include "icp_qat_fw_init_admin.h" #define ADF_ADMIN_MAILBOX_STRIDE 0x1000 @@ -15,6 +16,7 @@ #define ADF_CONST_TABLE_SIZE 1024 #define ADF_ADMIN_POLL_DELAY_US 20 #define ADF_ADMIN_POLL_TIMEOUT_US (5 * USEC_PER_SEC) +#define ADF_ONE_AE 1 static const u8 const_tab[1024] __aligned(1024) = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -194,6 +196,22 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) return adf_send_admin(accel_dev, &req, &resp, ae_mask); } +int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp) +{ + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp; + unsigned int ae_mask = ADF_ONE_AE; + int ret; + + req.cmd_id = ICP_QAT_FW_TIMER_GET; + ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); + if (ret) + return ret; + + *timestamp = resp.timestamp; + return 0; +} + static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev, u32 *capabilities) { @@ -223,6 +241,49 @@ static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev, return 0; } +int adf_get_ae_fw_counters(struct adf_accel_dev *accel_dev, u16 ae, u64 *reqs, u64 *resps) +{ + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + int ret; + + req.cmd_id = ICP_QAT_FW_COUNTERS_GET; + + ret = adf_put_admin_msg_sync(accel_dev, ae, &req, &resp); + if (ret || resp.status) + return -EFAULT; + + *reqs = resp.req_rec_count; + *resps = resp.resp_sent_count; + + return 0; +} + +int adf_send_admin_tim_sync(struct adf_accel_dev *accel_dev, u32 cnt) +{ + u32 ae_mask = accel_dev->hw_device->ae_mask; + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp = { }; + + req.cmd_id = ICP_QAT_FW_SYNC; + req.int_timer_ticks = cnt; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + +int adf_send_admin_hb_timer(struct adf_accel_dev *accel_dev, uint32_t ticks) +{ + u32 ae_mask = accel_dev->hw_device->ae_mask; + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp; + + req.cmd_id = ICP_QAT_FW_HEARTBEAT_TIMER_SET; + req.init_cfg_ptr = accel_dev->heartbeat->dma.phy_addr; + req.heartbeat_ticks = ticks; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + /** * adf_send_admin_init() - Function sends init message to FW * @accel_dev: Pointer to acceleration device. diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h index 3ae1e5caee0e..6066dc637352 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h @@ -47,4 +47,6 @@ #define ADF_ETRMGR_CORE_AFFINITY_FORMAT \ ADF_ETRMGR_BANK "%d" ADF_ETRMGR_CORE_AFFINITY #define ADF_ACCEL_STR "Accelerator%d" +#define ADF_HEARTBEAT_TIMER "HeartbeatTimer" + #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_clock.c b/drivers/crypto/intel/qat/qat_common/adf_clock.c new file mode 100644 index 000000000000..dc0778691eb0 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_clock.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include <linux/delay.h> +#include <linux/dev_printk.h> +#include <linux/export.h> +#include <linux/math.h> +#include <linux/minmax.h> +#include <linux/time64.h> +#include <linux/types.h> +#include <linux/units.h> +#include <asm/errno.h> +#include "adf_accel_devices.h" +#include "adf_clock.h" +#include "adf_common_drv.h" + +#define MEASURE_CLOCK_RETRIES 10 +#define MEASURE_CLOCK_DELAY_US 10000 +#define ME_CLK_DIVIDER 16 +#define MEASURE_CLOCK_DELTA_THRESHOLD_US 100 + +static inline u64 timespec_to_us(const struct timespec64 *ts) +{ + return (u64)DIV_ROUND_CLOSEST_ULL(timespec64_to_ns(ts), NSEC_PER_USEC); +} + +static inline u64 timespec_to_ms(const struct timespec64 *ts) +{ + return (u64)DIV_ROUND_CLOSEST_ULL(timespec64_to_ns(ts), NSEC_PER_MSEC); +} + +u64 adf_clock_get_current_time(void) +{ + struct timespec64 ts; + + ktime_get_real_ts64(&ts); + return timespec_to_ms(&ts); +} + +static int measure_clock(struct adf_accel_dev *accel_dev, u32 *frequency) +{ + struct timespec64 ts1, ts2, ts3, ts4; + u64 timestamp1, timestamp2, temp; + u32 delta_us, tries; + int ret; + + tries = MEASURE_CLOCK_RETRIES; + do { + ktime_get_real_ts64(&ts1); + ret = adf_get_fw_timestamp(accel_dev, ×tamp1); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to get fw timestamp\n"); + return ret; + } + ktime_get_real_ts64(&ts2); + delta_us = timespec_to_us(&ts2) - timespec_to_us(&ts1); + } while (delta_us > MEASURE_CLOCK_DELTA_THRESHOLD_US && --tries); + + if (!tries) { + dev_err(&GET_DEV(accel_dev), "Excessive clock measure delay\n"); + return -ETIMEDOUT; + } + + fsleep(MEASURE_CLOCK_DELAY_US); + + tries = MEASURE_CLOCK_RETRIES; + do { + ktime_get_real_ts64(&ts3); + if (adf_get_fw_timestamp(accel_dev, ×tamp2)) { + dev_err(&GET_DEV(accel_dev), + "Failed to get fw timestamp\n"); + return -EIO; + } + ktime_get_real_ts64(&ts4); + delta_us = timespec_to_us(&ts4) - timespec_to_us(&ts3); + } while (delta_us > MEASURE_CLOCK_DELTA_THRESHOLD_US && --tries); + + if (!tries) { + dev_err(&GET_DEV(accel_dev), "Excessive clock measure delay\n"); + return -ETIMEDOUT; + } + + delta_us = timespec_to_us(&ts3) - timespec_to_us(&ts1); + temp = (timestamp2 - timestamp1) * ME_CLK_DIVIDER * 10; + temp = DIV_ROUND_CLOSEST_ULL(temp, delta_us); + /* + * Enclose the division to allow the preprocessor to precalculate it, + * and avoid promoting r-value to 64-bit before division. + */ + *frequency = temp * (HZ_PER_MHZ / 10); + + return 0; +} + +/** + * adf_dev_measure_clock() - measures device clock frequency + * @accel_dev: Pointer to acceleration device. + * @frequency: Pointer to variable where result will be stored + * @min: Minimal allowed frequency value + * @max: Maximal allowed frequency value + * + * If the measurement result will go beyond the min/max thresholds the value + * will take the value of the crossed threshold. + * + * This algorithm compares the device firmware timestamp with the kernel + * timestamp. So we can't expect too high accuracy from this measurement. + * + * Return: + * * 0 - measurement succeed + * * -ETIMEDOUT - measurement failed + */ +int adf_dev_measure_clock(struct adf_accel_dev *accel_dev, + u32 *frequency, u32 min, u32 max) +{ + int ret; + u32 freq; + + ret = measure_clock(accel_dev, &freq); + if (ret) + return ret; + + *frequency = clamp(freq, min, max); + + if (*frequency != freq) + dev_warn(&GET_DEV(accel_dev), + "Measured clock %d Hz is out of range, assuming %d\n", + freq, *frequency); + return 0; +} +EXPORT_SYMBOL_GPL(adf_dev_measure_clock); diff --git a/drivers/crypto/intel/qat/qat_common/adf_clock.h b/drivers/crypto/intel/qat/qat_common/adf_clock.h new file mode 100644 index 000000000000..e309bc0dc35c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_clock.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_CLOCK_H +#define ADF_CLOCK_H + +#include <linux/types.h> + +struct adf_accel_dev; + +int adf_dev_measure_clock(struct adf_accel_dev *accel_dev, u32 *frequency, + u32 min, u32 max); +u64 adf_clock_get_current_time(void); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h index b8132eb9bc2a..673b5044c62a 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h @@ -58,12 +58,6 @@ int adf_dev_restart(struct adf_accel_dev *accel_dev); void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data); void adf_clean_vf_map(bool); - -int adf_ctl_dev_register(void); -void adf_ctl_dev_unregister(void); -int adf_processes_dev_register(void); -void adf_processes_dev_unregister(void); - int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, struct adf_accel_dev *pf); void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, @@ -94,7 +88,11 @@ void adf_exit_aer(void); int adf_init_admin_comms(struct adf_accel_dev *accel_dev); void adf_exit_admin_comms(struct adf_accel_dev *accel_dev); int adf_send_admin_init(struct adf_accel_dev *accel_dev); +int adf_get_ae_fw_counters(struct adf_accel_dev *accel_dev, u16 ae, u64 *reqs, u64 *resps); int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay); +int adf_send_admin_tim_sync(struct adf_accel_dev *accel_dev, u32 cnt); +int adf_send_admin_hb_timer(struct adf_accel_dev *accel_dev, uint32_t ticks); +int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp); int adf_init_arb(struct adf_accel_dev *accel_dev); void adf_exit_arb(struct adf_accel_dev *accel_dev); void adf_update_ring_arb(struct adf_etr_ring_data *ring); @@ -178,8 +176,6 @@ int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned long ctx_mask, unsigned short reg_num, unsigned int regdata); -int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned short lm_addr, unsigned int value); void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned char mode); int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); @@ -193,6 +189,8 @@ int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, int adf_init_misc_wq(void); void adf_exit_misc_wq(void); bool adf_misc_wq_queue_work(struct work_struct *work); +bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, + unsigned long delay); #if defined(CONFIG_PCI_IOV) int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c index d0a2f892e6eb..04845f8d72be 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c @@ -6,6 +6,8 @@ #include "adf_cfg.h" #include "adf_common_drv.h" #include "adf_dbgfs.h" +#include "adf_fw_counters.h" +#include "adf_heartbeat_dbgfs.h" /** * adf_dbgfs_init() - add persistent debugfs entries @@ -56,6 +58,11 @@ void adf_dbgfs_add(struct adf_accel_dev *accel_dev) { if (!accel_dev->debugfs_dir) return; + + if (!accel_dev->is_vf) { + adf_fw_counters_dbgfs_add(accel_dev); + adf_heartbeat_dbgfs_add(accel_dev); + } } /** @@ -66,4 +73,9 @@ void adf_dbgfs_rm(struct adf_accel_dev *accel_dev) { if (!accel_dev->debugfs_dir) return; + + if (!accel_dev->is_vf) { + adf_heartbeat_dbgfs_rm(accel_dev); + adf_fw_counters_dbgfs_rm(accel_dev); + } } diff --git a/drivers/crypto/intel/qat/qat_common/adf_fw_counters.c b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.c new file mode 100644 index 000000000000..cb6e09ef5c9f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include <linux/bitops.h> +#include <linux/debugfs.h> +#include <linux/err.h> +#include <linux/fs.h> +#include <linux/kernel.h> +#include <linux/seq_file.h> +#include <linux/types.h> + +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_fw_counters.h" + +#define ADF_FW_COUNTERS_MAX_PADDING 16 + +enum adf_fw_counters_types { + ADF_FW_REQUESTS, + ADF_FW_RESPONSES, + ADF_FW_COUNTERS_COUNT +}; + +static const char * const adf_fw_counter_names[] = { + [ADF_FW_REQUESTS] = "Requests", + [ADF_FW_RESPONSES] = "Responses", +}; + +static_assert(ARRAY_SIZE(adf_fw_counter_names) == ADF_FW_COUNTERS_COUNT); + +struct adf_ae_counters { + u16 ae; + u64 values[ADF_FW_COUNTERS_COUNT]; +}; + +struct adf_fw_counters { + u16 ae_count; + struct adf_ae_counters ae_counters[]; +}; + +static void adf_fw_counters_parse_ae_values(struct adf_ae_counters *ae_counters, u32 ae, + u64 req_count, u64 resp_count) +{ + ae_counters->ae = ae; + ae_counters->values[ADF_FW_REQUESTS] = req_count; + ae_counters->values[ADF_FW_RESPONSES] = resp_count; +} + +static int adf_fw_counters_load_from_device(struct adf_accel_dev *accel_dev, + struct adf_fw_counters *fw_counters) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + unsigned long ae_mask; + unsigned int i; + unsigned long ae; + + /* Ignore the admin AEs */ + ae_mask = hw_data->ae_mask & ~hw_data->admin_ae_mask; + + if (hweight_long(ae_mask) > fw_counters->ae_count) + return -EINVAL; + + i = 0; + for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { + u64 req_count, resp_count; + int ret; + + ret = adf_get_ae_fw_counters(accel_dev, ae, &req_count, &resp_count); + if (ret) + return ret; + + adf_fw_counters_parse_ae_values(&fw_counters->ae_counters[i++], ae, + req_count, resp_count); + } + + return 0; +} + +static struct adf_fw_counters *adf_fw_counters_allocate(unsigned long ae_count) +{ + struct adf_fw_counters *fw_counters; + + if (unlikely(!ae_count)) + return ERR_PTR(-EINVAL); + + fw_counters = kmalloc(struct_size(fw_counters, ae_counters, ae_count), GFP_KERNEL); + if (!fw_counters) + return ERR_PTR(-ENOMEM); + + fw_counters->ae_count = ae_count; + + return fw_counters; +} + +/** + * adf_fw_counters_get() - Return FW counters for the provided device. + * @accel_dev: Pointer to a QAT acceleration device + * + * Allocates and returns a table of counters containing execution statistics + * for each non-admin AE available through the supplied acceleration device. + * The caller becomes the owner of such memory and is responsible for + * the deallocation through a call to kfree(). + * + * Returns: a pointer to a dynamically allocated struct adf_fw_counters + * on success, or a negative value on error. + */ +static struct adf_fw_counters *adf_fw_counters_get(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_fw_counters *fw_counters; + unsigned long ae_count; + int ret; + + if (!adf_dev_started(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "QAT Device not started\n"); + return ERR_PTR(-EFAULT); + } + + /* Ignore the admin AEs */ + ae_count = hweight_long(hw_data->ae_mask & ~hw_data->admin_ae_mask); + + fw_counters = adf_fw_counters_allocate(ae_count); + if (IS_ERR(fw_counters)) + return fw_counters; + + ret = adf_fw_counters_load_from_device(accel_dev, fw_counters); + if (ret) { + kfree(fw_counters); + dev_err(&GET_DEV(accel_dev), + "Failed to create QAT fw_counters file table [%d].\n", ret); + return ERR_PTR(ret); + } + + return fw_counters; +} + +static void *qat_fw_counters_seq_start(struct seq_file *sfile, loff_t *pos) +{ + struct adf_fw_counters *fw_counters = sfile->private; + + if (*pos == 0) + return SEQ_START_TOKEN; + + if (*pos > fw_counters->ae_count) + return NULL; + + return &fw_counters->ae_counters[*pos - 1]; +} + +static void *qat_fw_counters_seq_next(struct seq_file *sfile, void *v, loff_t *pos) +{ + struct adf_fw_counters *fw_counters = sfile->private; + + (*pos)++; + + if (*pos > fw_counters->ae_count) + return NULL; + + return &fw_counters->ae_counters[*pos - 1]; +} + +static void qat_fw_counters_seq_stop(struct seq_file *sfile, void *v) {} + +static int qat_fw_counters_seq_show(struct seq_file *sfile, void *v) +{ + int i; + + if (v == SEQ_START_TOKEN) { + seq_puts(sfile, "AE "); + for (i = 0; i < ADF_FW_COUNTERS_COUNT; ++i) + seq_printf(sfile, " %*s", ADF_FW_COUNTERS_MAX_PADDING, + adf_fw_counter_names[i]); + } else { + struct adf_ae_counters *ae_counters = (struct adf_ae_counters *)v; + + seq_printf(sfile, "%2d:", ae_counters->ae); + for (i = 0; i < ADF_FW_COUNTERS_COUNT; ++i) + seq_printf(sfile, " %*llu", ADF_FW_COUNTERS_MAX_PADDING, + ae_counters->values[i]); + } + seq_putc(sfile, '\n'); + + return 0; +} + +static const struct seq_operations qat_fw_counters_sops = { + .start = qat_fw_counters_seq_start, + .next = qat_fw_counters_seq_next, + .stop = qat_fw_counters_seq_stop, + .show = qat_fw_counters_seq_show, +}; + +static int qat_fw_counters_file_open(struct inode *inode, struct file *file) +{ + struct adf_accel_dev *accel_dev = inode->i_private; + struct seq_file *fw_counters_seq_file; + struct adf_fw_counters *fw_counters; + int ret; + + fw_counters = adf_fw_counters_get(accel_dev); + if (IS_ERR(fw_counters)) + return PTR_ERR(fw_counters); + + ret = seq_open(file, &qat_fw_counters_sops); + if (unlikely(ret)) { + kfree(fw_counters); + return ret; + } + + fw_counters_seq_file = file->private_data; + fw_counters_seq_file->private = fw_counters; + return ret; +} + +static int qat_fw_counters_file_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + + kfree(seq->private); + seq->private = NULL; + + return seq_release(inode, file); } + +static const struct file_operations qat_fw_counters_fops = { + .owner = THIS_MODULE, + .open = qat_fw_counters_file_open, + .read = seq_read, + .llseek = seq_lseek, + .release = qat_fw_counters_file_release, +}; + +/** + * adf_fw_counters_dbgfs_add() - Create a debugfs file containing FW + * execution counters. + * @accel_dev: Pointer to a QAT acceleration device + * + * Function creates a file to display a table with statistics for the given + * QAT acceleration device. The table stores device specific execution values + * for each AE, such as the number of requests sent to the FW and responses + * received from the FW. + * + * Return: void + */ +void adf_fw_counters_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + accel_dev->fw_cntr_dbgfile = debugfs_create_file("fw_counters", 0400, + accel_dev->debugfs_dir, + accel_dev, + &qat_fw_counters_fops); +} + +/** + * adf_fw_counters_dbgfs_rm() - Remove the debugfs file containing FW counters. + * @accel_dev: Pointer to a QAT acceleration device. + * + * Function removes the file providing the table of statistics for the given + * QAT acceleration device. + * + * Return: void + */ +void adf_fw_counters_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + debugfs_remove(accel_dev->fw_cntr_dbgfile); + accel_dev->fw_cntr_dbgfile = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_fw_counters.h b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.h new file mode 100644 index 000000000000..91b3b6a95f1f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_FW_COUNTERS_H +#define ADF_FW_COUNTERS_H + +struct adf_accel_dev; + +void adf_fw_counters_dbgfs_add(struct adf_accel_dev *accel_dev); +void adf_fw_counters_dbgfs_rm(struct adf_accel_dev *accel_dev); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c index eeb30da7587a..c27ff6d18e11 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c @@ -7,6 +7,7 @@ #include "adf_common_drv.h" #include "qat_crypto.h" #include "qat_compression.h" +#include "adf_heartbeat.h" #include "adf_transport_access_macros.h" static int adf_gen2_crypto_dev_config(struct adf_accel_dev *accel_dev) @@ -195,6 +196,12 @@ int adf_gen2_dev_config(struct adf_accel_dev *accel_dev) if (ret) goto err; + ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); + if (ret) + goto err; + + adf_heartbeat_save_cfg_param(accel_dev, ADF_CFG_HB_TIMER_DEFAULT_MS); + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); return ret; diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h index e4bc07529be4..6bd341061de4 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h @@ -145,6 +145,9 @@ do { \ #define ADF_GEN2_CERRSSMSH(i) ((i) * 0x4000 + 0x10) #define ADF_GEN2_ERRSSMSH_EN BIT(3) +/* Number of heartbeat counter pairs */ +#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE + /* Interrupts */ #define ADF_GEN2_SMIAPF0_MASK_OFFSET (0x3A000 + 0x28) #define ADF_GEN2_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h index 4fb4b3df5a18..02d7a019ebf8 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h @@ -136,6 +136,9 @@ do { \ #define ADF_GEN4_VFLNOTIFY BIT(7) +/* Number of heartbeat counter pairs */ +#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE + void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h index dd112923e006..c2768762cca3 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h @@ -35,7 +35,7 @@ #define ADF_GEN4_PM_MSG_PENDING BIT(0) #define ADF_GEN4_PM_MSG_PAYLOAD_BIT_MASK GENMASK(28, 1) -#define ADF_GEN4_PM_DEFAULT_IDLE_FILTER (0x0) +#define ADF_GEN4_PM_DEFAULT_IDLE_FILTER (0x6) #define ADF_GEN4_PM_MAX_IDLE_FILTER (0x7) #define ADF_GEN4_PM_DEFAULT_IDLE_SUPPORT (0x1) diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c new file mode 100644 index 000000000000..646c57922fcd --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include <linux/container_of.h> +#include <linux/dev_printk.h> +#include <linux/export.h> +#include <linux/jiffies.h> +#include <linux/ktime.h> +#include <linux/slab.h> +#include <linux/workqueue.h> + +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_gen4_timer.h" + +#define ADF_GEN4_TIMER_PERIOD_MS 200 + +/* This periodic update is used to trigger HB, RL & TL fw events */ +static void work_handler(struct work_struct *work) +{ + struct adf_accel_dev *accel_dev; + struct adf_timer *timer_ctx; + u32 time_periods; + + timer_ctx = container_of(to_delayed_work(work), struct adf_timer, work_ctx); + accel_dev = timer_ctx->accel_dev; + + adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, + msecs_to_jiffies(ADF_GEN4_TIMER_PERIOD_MS)); + + time_periods = div_u64(ktime_ms_delta(ktime_get_real(), timer_ctx->initial_ktime), + ADF_GEN4_TIMER_PERIOD_MS); + + if (adf_send_admin_tim_sync(accel_dev, time_periods)) + dev_err(&GET_DEV(accel_dev), "Failed to synchronize qat timer\n"); +} + +int adf_gen4_timer_start(struct adf_accel_dev *accel_dev) +{ + struct adf_timer *timer_ctx; + + timer_ctx = kzalloc(sizeof(*timer_ctx), GFP_KERNEL); + if (!timer_ctx) + return -ENOMEM; + + timer_ctx->accel_dev = accel_dev; + accel_dev->timer = timer_ctx; + timer_ctx->initial_ktime = ktime_get_real(); + + INIT_DELAYED_WORK(&timer_ctx->work_ctx, work_handler); + adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, + msecs_to_jiffies(ADF_GEN4_TIMER_PERIOD_MS)); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_timer_start); + +void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev) +{ + struct adf_timer *timer_ctx = accel_dev->timer; + + if (!timer_ctx) + return; + + cancel_delayed_work_sync(&timer_ctx->work_ctx); + + kfree(timer_ctx); + accel_dev->timer = NULL; +} +EXPORT_SYMBOL_GPL(adf_gen4_timer_stop); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h new file mode 100644 index 000000000000..66a709e7b358 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_GEN4_TIMER_H_ +#define ADF_GEN4_TIMER_H_ + +#include <linux/ktime.h> +#include <linux/workqueue.h> + +struct adf_accel_dev; + +struct adf_timer { + struct adf_accel_dev *accel_dev; + struct delayed_work work_ctx; + ktime_t initial_ktime; +}; + +int adf_gen4_timer_start(struct adf_accel_dev *accel_dev); +void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev); + +#endif /* ADF_GEN4_TIMER_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat.c b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.c new file mode 100644 index 000000000000..beef9a5f6c75 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.c @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include <linux/dev_printk.h> +#include <linux/dma-mapping.h> +#include <linux/export.h> +#include <linux/kernel.h> +#include <linux/kstrtox.h> +#include <linux/overflow.h> +#include <linux/string.h> +#include <linux/slab.h> +#include <linux/types.h> +#include <asm/errno.h> +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "adf_clock.h" +#include "adf_common_drv.h" +#include "adf_heartbeat.h" +#include "adf_transport_internal.h" +#include "icp_qat_fw_init_admin.h" + +#define ADF_HB_EMPTY_SIG 0xA5A5A5A5 + +/* Heartbeat counter pair */ +struct hb_cnt_pair { + __u16 resp_heartbeat_cnt; + __u16 req_heartbeat_cnt; +}; + +static int adf_hb_check_polling_freq(struct adf_accel_dev *accel_dev) +{ + u64 curr_time = adf_clock_get_current_time(); + u64 polling_time = curr_time - accel_dev->heartbeat->last_hb_check_time; + + if (polling_time < accel_dev->heartbeat->hb_timer) { + dev_warn(&GET_DEV(accel_dev), + "HB polling too frequent. Configured HB timer %d ms\n", + accel_dev->heartbeat->hb_timer); + return -EINVAL; + } + + accel_dev->heartbeat->last_hb_check_time = curr_time; + return 0; +} + +/** + * validate_hb_ctrs_cnt() - checks if the number of heartbeat counters should + * be updated by one to support the currently loaded firmware. + * @accel_dev: Pointer to acceleration device. + * + * Return: + * * true - hb_ctrs must increased by ADF_NUM_PKE_STRAND + * * false - no changes needed + */ +static bool validate_hb_ctrs_cnt(struct adf_accel_dev *accel_dev) +{ + const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs; + const size_t max_aes = accel_dev->hw_device->num_engines; + const size_t hb_struct_size = sizeof(struct hb_cnt_pair); + const size_t exp_diff_size = array3_size(ADF_NUM_PKE_STRAND, max_aes, + hb_struct_size); + const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); + const size_t stats_size = size_mul(dev_ctrs, hb_struct_size); + const u32 exp_diff_cnt = exp_diff_size / sizeof(u32); + const u32 stats_el_cnt = stats_size / sizeof(u32); + struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr; + const u32 *mem_to_chk = (u32 *)(hb_stats + dev_ctrs); + u32 el_diff_cnt = 0; + int i; + + /* count how many bytes are different from pattern */ + for (i = 0; i < stats_el_cnt; i++) { + if (mem_to_chk[i] == ADF_HB_EMPTY_SIG) + break; + + el_diff_cnt++; + } + + return el_diff_cnt && el_diff_cnt == exp_diff_cnt; +} + +void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev) +{ + struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr; + const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs; + const size_t max_aes = accel_dev->hw_device->num_engines; + const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); + const size_t stats_size = size_mul(dev_ctrs, sizeof(struct hb_cnt_pair)); + const size_t mem_items_to_fill = size_mul(stats_size, 2) / sizeof(u32); + + /* fill hb stats memory with pattern */ + memset32((uint32_t *)hb_stats, ADF_HB_EMPTY_SIG, mem_items_to_fill); + accel_dev->heartbeat->ctrs_cnt_checked = false; +} +EXPORT_SYMBOL_GPL(adf_heartbeat_check_ctrs); + +static int get_timer_ticks(struct adf_accel_dev *accel_dev, unsigned int *value) +{ + char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; + u32 timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS; + int cfg_read_status; + u32 ticks; + int ret; + + cfg_read_status = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_HEARTBEAT_TIMER, timer_str); + if (cfg_read_status == 0) { + if (kstrtouint(timer_str, 10, &timer_ms)) + dev_dbg(&GET_DEV(accel_dev), + "kstrtouint failed to parse the %s, param value", + ADF_HEARTBEAT_TIMER); + } + + if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) { + dev_err(&GET_DEV(accel_dev), "Timer cannot be less than %u\n", + ADF_CFG_HB_TIMER_MIN_MS); + return -EINVAL; + } + + /* + * On 4xxx devices adf_timer is responsible for HB updates and + * its period is fixed to 200ms + */ + if (accel_dev->timer) + timer_ms = ADF_CFG_HB_TIMER_MIN_MS; + + ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks); + if (ret) + return ret; + + adf_heartbeat_save_cfg_param(accel_dev, timer_ms); + + accel_dev->heartbeat->hb_timer = timer_ms; + *value = ticks; + + return 0; +} + +static int check_ae(struct hb_cnt_pair *curr, struct hb_cnt_pair *prev, + u16 *count, const size_t hb_ctrs) +{ + size_t thr; + + /* loop through all threads in AE */ + for (thr = 0; thr < hb_ctrs; thr++) { + u16 req = curr[thr].req_heartbeat_cnt; + u16 resp = curr[thr].resp_heartbeat_cnt; + u16 last = prev[thr].resp_heartbeat_cnt; + + if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && resp == last) { + u16 retry = ++count[thr]; + + if (retry >= ADF_CFG_HB_COUNT_THRESHOLD) + return -EIO; + + } else { + count[thr] = 0; + } + } + return 0; +} + +static int adf_hb_get_status(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct hb_cnt_pair *live_stats, *last_stats, *curr_stats; + const size_t hb_ctrs = hw_device->num_hb_ctrs; + const unsigned long ae_mask = hw_device->ae_mask; + const size_t max_aes = hw_device->num_engines; + const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); + const size_t stats_size = size_mul(dev_ctrs, sizeof(*curr_stats)); + struct hb_cnt_pair *ae_curr_p, *ae_prev_p; + u16 *count_fails, *ae_count_p; + size_t ae_offset; + size_t ae = 0; + int ret = 0; + + if (!accel_dev->heartbeat->ctrs_cnt_checked) { + if (validate_hb_ctrs_cnt(accel_dev)) + hw_device->num_hb_ctrs += ADF_NUM_PKE_STRAND; + + accel_dev->heartbeat->ctrs_cnt_checked = true; + } + + live_stats = accel_dev->heartbeat->dma.virt_addr; + last_stats = live_stats + dev_ctrs; + count_fails = (u16 *)(last_stats + dev_ctrs); + + curr_stats = kmemdup(live_stats, stats_size, GFP_KERNEL); + if (!curr_stats) + return -ENOMEM; + + /* loop through active AEs */ + for_each_set_bit(ae, &ae_mask, max_aes) { + ae_offset = size_mul(ae, hb_ctrs); + ae_curr_p = curr_stats + ae_offset; + ae_prev_p = last_stats + ae_offset; + ae_count_p = count_fails + ae_offset; + + ret = check_ae(ae_curr_p, ae_prev_p, ae_count_p, hb_ctrs); + if (ret) + break; + } + + /* Copy current stats for the next iteration */ + memcpy(last_stats, curr_stats, stats_size); + kfree(curr_stats); + + return ret; +} + +void adf_heartbeat_status(struct adf_accel_dev *accel_dev, + enum adf_device_heartbeat_status *hb_status) +{ + struct adf_heartbeat *hb; + + if (!adf_dev_started(accel_dev) || + test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) { + *hb_status = HB_DEV_UNRESPONSIVE; + return; + } + + if (adf_hb_check_polling_freq(accel_dev) == -EINVAL) { + *hb_status = HB_DEV_UNSUPPORTED; + return; + } + + hb = accel_dev->heartbeat; + hb->hb_sent_counter++; + + if (adf_hb_get_status(accel_dev)) { + dev_err(&GET_DEV(accel_dev), + "Heartbeat ERROR: QAT is not responding.\n"); + *hb_status = HB_DEV_UNRESPONSIVE; + hb->hb_failed_counter++; + return; + } + + *hb_status = HB_DEV_ALIVE; +} + +int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms, + u32 *value) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 clk_per_sec; + + /* HB clock may be different than AE clock */ + if (!hw_data->get_hb_clock) + return -EINVAL; + + clk_per_sec = hw_data->get_hb_clock(hw_data); + *value = time_ms * (clk_per_sec / MSEC_PER_SEC); + + return 0; +} + +int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, + unsigned int timer_ms) +{ + char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + + snprintf(timer_str, sizeof(timer_str), "%u", timer_ms); + return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_HEARTBEAT_TIMER, timer_str, + ADF_STR); +} +EXPORT_SYMBOL_GPL(adf_heartbeat_save_cfg_param); + +int adf_heartbeat_init(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb; + + hb = kzalloc(sizeof(*hb), GFP_KERNEL); + if (!hb) + goto err_ret; + + hb->dma.virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + &hb->dma.phy_addr, GFP_KERNEL); + if (!hb->dma.virt_addr) + goto err_free; + + /* + * Default set this flag as true to avoid unnecessary checks, + * it will be reset on platforms that need such a check + */ + hb->ctrs_cnt_checked = true; + accel_dev->heartbeat = hb; + + return 0; + +err_free: + kfree(hb); +err_ret: + return -ENOMEM; +} + +int adf_heartbeat_start(struct adf_accel_dev *accel_dev) +{ + unsigned int timer_ticks; + int ret; + + if (!accel_dev->heartbeat) { + dev_warn(&GET_DEV(accel_dev), "Heartbeat instance not found!"); + return -EFAULT; + } + + if (accel_dev->hw_device->check_hb_ctrs) + accel_dev->hw_device->check_hb_ctrs(accel_dev); + + ret = get_timer_ticks(accel_dev, &timer_ticks); + if (ret) + return ret; + + ret = adf_send_admin_hb_timer(accel_dev, timer_ticks); + if (ret) + dev_warn(&GET_DEV(accel_dev), "Heartbeat not supported!"); + + return ret; +} + +void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb = accel_dev->heartbeat; + + if (!hb) + return; + + if (hb->dma.virt_addr) + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + hb->dma.virt_addr, hb->dma.phy_addr); + + kfree(hb); + accel_dev->heartbeat = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat.h b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.h new file mode 100644 index 000000000000..b22e3cb29798 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_HEARTBEAT_H_ +#define ADF_HEARTBEAT_H_ + +#include <linux/types.h> + +struct adf_accel_dev; +struct dentry; + +#define ADF_CFG_HB_TIMER_MIN_MS 200 +#define ADF_CFG_HB_TIMER_DEFAULT_MS 500 +#define ADF_CFG_HB_COUNT_THRESHOLD 3 + +enum adf_device_heartbeat_status { + HB_DEV_UNRESPONSIVE = 0, + HB_DEV_ALIVE, + HB_DEV_UNSUPPORTED, +}; + +struct adf_heartbeat { + unsigned int hb_sent_counter; + unsigned int hb_failed_counter; + unsigned int hb_timer; + u64 last_hb_check_time; + bool ctrs_cnt_checked; + struct hb_dma_addr { + dma_addr_t phy_addr; + void *virt_addr; + } dma; + struct { + struct dentry *base_dir; + struct dentry *status; + struct dentry *cfg; + struct dentry *sent; + struct dentry *failed; + } dbgfs; +}; + +#ifdef CONFIG_DEBUG_FS +int adf_heartbeat_init(struct adf_accel_dev *accel_dev); +int adf_heartbeat_start(struct adf_accel_dev *accel_dev); +void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev); + +int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms, + uint32_t *value); +int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, + unsigned int timer_ms); +void adf_heartbeat_status(struct adf_accel_dev *accel_dev, + enum adf_device_heartbeat_status *hb_status); +void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev); + +#else +static inline int adf_heartbeat_init(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline int adf_heartbeat_start(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev) +{ +} + +static inline int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, + unsigned int timer_ms) +{ + return 0; +} + +static inline void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev) +{ +} +#endif +#endif /* ADF_HEARTBEAT_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c new file mode 100644 index 000000000000..803cbfd838f0 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include <linux/debugfs.h> +#include <linux/errno.h> +#include <linux/export.h> +#include <linux/fs.h> +#include <linux/kernel.h> +#include <linux/kstrtox.h> +#include <linux/types.h> +#include "adf_cfg.h" +#include "adf_common_drv.h" +#include "adf_heartbeat.h" +#include "adf_heartbeat_dbgfs.h" + +#define HB_OK 0 +#define HB_ERROR -1 +#define HB_STATUS_MAX_STRLEN 4 +#define HB_STATS_MAX_STRLEN 16 + +static ssize_t adf_hb_stats_read(struct file *file, char __user *user_buffer, + size_t count, loff_t *ppos) +{ + char buf[HB_STATS_MAX_STRLEN]; + unsigned int *value; + int len; + + if (*ppos > 0) + return 0; + + value = file->private_data; + len = scnprintf(buf, sizeof(buf), "%u\n", *value); + + return simple_read_from_buffer(user_buffer, count, ppos, buf, len + 1); +} + +static const struct file_operations adf_hb_stats_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = adf_hb_stats_read, +}; + +static ssize_t adf_hb_status_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + enum adf_device_heartbeat_status hb_status; + char ret_str[HB_STATUS_MAX_STRLEN]; + struct adf_accel_dev *accel_dev; + int ret_code; + size_t len; + + if (*ppos > 0) + return 0; + + accel_dev = file->private_data; + ret_code = HB_OK; + + adf_heartbeat_status(accel_dev, &hb_status); + + if (hb_status != HB_DEV_ALIVE) + ret_code = HB_ERROR; + + len = scnprintf(ret_str, sizeof(ret_str), "%d\n", ret_code); + + return simple_read_from_buffer(user_buf, count, ppos, ret_str, len + 1); +} + +static const struct file_operations adf_hb_status_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = adf_hb_status_read, +}; + +static ssize_t adf_hb_cfg_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + struct adf_accel_dev *accel_dev; + unsigned int timer_ms; + int len; + + if (*ppos > 0) + return 0; + + accel_dev = file->private_data; + timer_ms = accel_dev->heartbeat->hb_timer; + len = scnprintf(timer_str, sizeof(timer_str), "%u\n", timer_ms); + + return simple_read_from_buffer(user_buf, count, ppos, timer_str, + len + 1); +} + +static ssize_t adf_hb_cfg_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + char input_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; + struct adf_accel_dev *accel_dev; + int ret, written_chars; + unsigned int timer_ms; + u32 ticks; + + accel_dev = file->private_data; + timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS; + + /* last byte left as string termination */ + if (count > sizeof(input_str) - 1) + return -EINVAL; + + written_chars = simple_write_to_buffer(input_str, sizeof(input_str) - 1, + ppos, user_buf, count); + if (written_chars > 0) { + ret = kstrtouint(input_str, 10, &timer_ms); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "heartbeat_cfg: Invalid value\n"); + return ret; + } + + if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) { + dev_err(&GET_DEV(accel_dev), + "heartbeat_cfg: Invalid value\n"); + return -EINVAL; + } + + /* + * On 4xxx devices adf_timer is responsible for HB updates and + * its period is fixed to 200ms + */ + if (accel_dev->timer) + timer_ms = ADF_CFG_HB_TIMER_MIN_MS; + + ret = adf_heartbeat_save_cfg_param(accel_dev, timer_ms); + if (ret) + return ret; + + ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks); + if (ret) + return ret; + + ret = adf_send_admin_hb_timer(accel_dev, ticks); + if (ret) + return ret; + + accel_dev->heartbeat->hb_timer = timer_ms; + } + + return written_chars; +} + +static const struct file_operations adf_hb_cfg_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = adf_hb_cfg_read, + .write = adf_hb_cfg_write, +}; + +void adf_heartbeat_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb = accel_dev->heartbeat; + + if (!hb) + return; + + hb->dbgfs.base_dir = debugfs_create_dir("heartbeat", accel_dev->debugfs_dir); + hb->dbgfs.status = debugfs_create_file("status", 0400, hb->dbgfs.base_dir, + accel_dev, &adf_hb_status_fops); + hb->dbgfs.sent = debugfs_create_file("queries_sent", 0400, hb->dbgfs.base_dir, + &hb->hb_sent_counter, &adf_hb_stats_fops); + hb->dbgfs.failed = debugfs_create_file("queries_failed", 0400, hb->dbgfs.base_dir, + &hb->hb_failed_counter, &adf_hb_stats_fops); + hb->dbgfs.cfg = debugfs_create_file("config", 0600, hb->dbgfs.base_dir, + accel_dev, &adf_hb_cfg_fops); +} +EXPORT_SYMBOL_GPL(adf_heartbeat_dbgfs_add); + +void adf_heartbeat_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb = accel_dev->heartbeat; + + if (!hb) + return; + + debugfs_remove(hb->dbgfs.status); + hb->dbgfs.status = NULL; + debugfs_remove(hb->dbgfs.sent); + hb->dbgfs.sent = NULL; + debugfs_remove(hb->dbgfs.failed); + hb->dbgfs.failed = NULL; + debugfs_remove(hb->dbgfs.cfg); + hb->dbgfs.cfg = NULL; + debugfs_remove(hb->dbgfs.base_dir); + hb->dbgfs.base_dir = NULL; +} +EXPORT_SYMBOL_GPL(adf_heartbeat_dbgfs_rm); diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.h b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.h new file mode 100644 index 000000000000..84dd29ea6454 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_HEARTBEAT_DBGFS_H_ +#define ADF_HEARTBEAT_DBGFS_H_ + +struct adf_accel_dev; + +void adf_heartbeat_dbgfs_add(struct adf_accel_dev *accel_dev); +void adf_heartbeat_dbgfs_rm(struct adf_accel_dev *accel_dev); + +#endif /* ADF_HEARTBEAT_DBGFS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_init.c b/drivers/crypto/intel/qat/qat_common/adf_init.c index 826179c98524..89001fe92e76 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_init.c +++ b/drivers/crypto/intel/qat/qat_common/adf_init.c @@ -8,6 +8,7 @@ #include "adf_cfg.h" #include "adf_common_drv.h" #include "adf_dbgfs.h" +#include "adf_heartbeat.h" static LIST_HEAD(service_table); static DEFINE_MUTEX(service_lock); @@ -129,6 +130,8 @@ static int adf_dev_init(struct adf_accel_dev *accel_dev) return -EFAULT; } + adf_heartbeat_init(accel_dev); + /* * Subservice initialisation is divided into two stages: init and start. * This is to facilitate any ordering dependencies between services @@ -163,6 +166,7 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct service_hndl *service; struct list_head *list_itr; + int ret; set_bit(ADF_STATUS_STARTING, &accel_dev->status); @@ -177,6 +181,14 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) return -EFAULT; } + if (hw_data->measure_clock) { + ret = hw_data->measure_clock(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed measure device clock\n"); + return ret; + } + } + /* Set ssm watch dog timer */ if (hw_data->set_ssm_wdtimer) hw_data->set_ssm_wdtimer(accel_dev); @@ -187,6 +199,16 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) return -EFAULT; } + if (hw_data->start_timer) { + ret = hw_data->start_timer(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to start internal sync timer\n"); + return ret; + } + } + + adf_heartbeat_start(accel_dev); + list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); if (service->event_hld(accel_dev, ADF_EVENT_START)) { @@ -235,6 +257,7 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) */ static void adf_dev_stop(struct adf_accel_dev *accel_dev) { + struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct service_hndl *service; struct list_head *list_itr; bool wait = false; @@ -270,6 +293,9 @@ static void adf_dev_stop(struct adf_accel_dev *accel_dev) } } + if (hw_data->stop_timer) + hw_data->stop_timer(accel_dev); + if (wait) msleep(100); @@ -326,6 +352,8 @@ static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) clear_bit(accel_dev->accel_id, service->init_status); } + adf_heartbeat_shutdown(accel_dev); + hw_data->disable_iov(accel_dev); if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_isr.c index ad9e135b8560..2aba194a7c29 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_isr.c @@ -380,3 +380,9 @@ bool adf_misc_wq_queue_work(struct work_struct *work) { return queue_work(adf_misc_wq, work); } + +bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, + unsigned long delay) +{ + return queue_delayed_work(adf_misc_wq, work, delay); +} diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h index 56cb827f93ea..3e968a4bcc9c 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h @@ -16,6 +16,8 @@ enum icp_qat_fw_init_admin_cmd_id { ICP_QAT_FW_HEARTBEAT_SYNC = 7, ICP_QAT_FW_HEARTBEAT_GET = 8, ICP_QAT_FW_COMP_CAPABILITY_GET = 9, + ICP_QAT_FW_HEARTBEAT_TIMER_SET = 13, + ICP_QAT_FW_TIMER_GET = 19, ICP_QAT_FW_PM_STATE_CONFIG = 128, }; @@ -37,6 +39,12 @@ struct icp_qat_fw_init_admin_req { __u16 ibuf_size_in_kb; __u16 resrvd3; }; + struct { + __u32 int_timer_ticks; + }; + struct { + __u32 heartbeat_ticks; + }; __u32 idle_filter; }; @@ -97,19 +105,6 @@ struct icp_qat_fw_init_admin_resp { }; } __packed; -#define ICP_QAT_FW_COMN_HEARTBEAT_OK 0 -#define ICP_QAT_FW_COMN_HEARTBEAT_BLOCKED 1 -#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS 0 -#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_MASK 0x1 -#define ICP_QAT_FW_COMN_STATUS_RESRVD_FLD_MASK 0xFE -#define ICP_QAT_FW_COMN_HEARTBEAT_HDR_FLAG_GET(hdr_t) \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_GET(hdr_t.flags) - -#define ICP_QAT_FW_COMN_HEARTBEAT_HDR_FLAG_SET(hdr_t, val) \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_SET(hdr_t, val) +#define ICP_QAT_FW_SYNC ICP_QAT_FW_HEARTBEAT_SYNC -#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_GET(flags) \ - QAT_FIELD_GET(flags, \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS, \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_MASK) #endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h index a65059e56248..0c8883e2ccc6 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h @@ -97,7 +97,10 @@ enum icp_qat_capabilities_mask { ICP_ACCEL_CAPABILITIES_SHA3_EXT = BIT(15), ICP_ACCEL_CAPABILITIES_AESGCM_SPC = BIT(16), ICP_ACCEL_CAPABILITIES_CHACHA_POLY = BIT(17), - /* Bits 18-21 are currently reserved */ + ICP_ACCEL_CAPABILITIES_SM2 = BIT(18), + ICP_ACCEL_CAPABILITIES_SM3 = BIT(19), + ICP_ACCEL_CAPABILITIES_SM4 = BIT(20), + /* Bit 21 is currently reserved */ ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY = BIT(22), ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64 = BIT(23), ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION = BIT(24), diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c index 3f1f35283266..7842a9f22178 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.c +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c @@ -234,8 +234,7 @@ static void qat_free_dc_data(struct adf_accel_dev *accel_dev) dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz, DMA_FROM_DEVICE); - memset(dc_data->ovf_buff, 0, dc_data->ovf_buff_sz); - kfree(dc_data->ovf_buff); + kfree_sensitive(dc_data->ovf_buff); devm_kfree(dev, dc_data); accel_dev->dc_data = NULL; } diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index ce837bcc1cab..4bd150d1441a 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -11,7 +11,7 @@ #include "icp_qat_hal.h" #include "icp_qat_fw_loader_handle.h" -#define UWORD_CPYBUF_SIZE 1024 +#define UWORD_CPYBUF_SIZE 1024U #define INVLD_UWORD 0xffffffffffull #define PID_MINOR_REV 0xf #define PID_MAJOR_REV (0xf << 4) @@ -1986,10 +1986,7 @@ static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, uw_relative_addr = 0; words_num = encap_page->micro_words_num; while (words_num) { - if (words_num < UWORD_CPYBUF_SIZE) - cpylen = words_num; - else - cpylen = UWORD_CPYBUF_SIZE; + cpylen = min(words_num, UWORD_CPYBUF_SIZE); /* load the buffer */ for (i = 0; i < cpylen; i++) |