From ecc758cee6a1088f2e2e684a09fea650472a47f0 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Mon, 24 Apr 2023 08:53:55 +0200 Subject: s390/pai_crypto: replace atomic_t with refcount_t The s390 PMU of PAI crypto counters uses atomic_t for reference counting. Replace this with the proper data type refcount_t. No functional change. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Signed-off-by: Alexander Gordeev --- arch/s390/kernel/perf_pai_crypto.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index a7b339c4fd7c..fe7d1774ded1 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -36,7 +36,7 @@ struct paicrypt_map { unsigned long *page; /* Page for CPU to store counters */ struct pai_userdata *save; /* Page to store no-zero counters */ unsigned int active_events; /* # of PAI crypto users */ - unsigned int refcnt; /* Reference count mapped buffers */ + refcount_t refcnt; /* Reference count mapped buffers */ enum paievt_mode mode; /* Type of event */ struct perf_event *event; /* Perf event for sampling */ }; @@ -57,10 +57,11 @@ static void paicrypt_event_destroy(struct perf_event *event) static_branch_dec(&pai_key); mutex_lock(&pai_reserve_mutex); debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d" - " mode %d refcnt %d\n", __func__, + " mode %d refcnt %u\n", __func__, event->attr.config, event->cpu, - cpump->active_events, cpump->mode, cpump->refcnt); - if (!--cpump->refcnt) { + cpump->active_events, cpump->mode, + refcount_read(&cpump->refcnt)); + if (refcount_dec_and_test(&cpump->refcnt)) { debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n", __func__, (unsigned long)cpump->page, cpump->save); @@ -149,8 +150,10 @@ static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump) /* Allocate memory for counter page and counter extraction. * Only the first counting event has to allocate a page. */ - if (cpump->page) + if (cpump->page) { + refcount_inc(&cpump->refcnt); goto unlock; + } rc = -ENOMEM; cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL); @@ -164,18 +167,18 @@ static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump) goto unlock; } rc = 0; + refcount_set(&cpump->refcnt, 1); unlock: /* If rc is non-zero, do not set mode and reference count */ if (!rc) { - cpump->refcnt++; cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING; } debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d" - " mode %d refcnt %d page %#lx save %p rc %d\n", + " mode %d refcnt %u page %#lx save %p rc %d\n", __func__, a->sample_period, cpump->active_events, - cpump->mode, cpump->refcnt, + cpump->mode, refcount_read(&cpump->refcnt), (unsigned long)cpump->page, cpump->save, rc); mutex_unlock(&pai_reserve_mutex); return rc; -- cgit v1.2.3 From 1f2597cd3686955a4d64e01909dbfe625a2a35a1 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 27 Apr 2023 10:33:31 +0200 Subject: s390/pai_ext: replace atomic_t with refcount_t The s390 PMU of PAI extension 1 NNPA counters uses atomic_t for reference counting. Replace this with the proper data type refcount_t. No functional change. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Signed-off-by: Alexander Gordeev --- arch/s390/kernel/perf_pai_ext.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index fcea307d7529..3b4f384f77f7 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -50,7 +50,7 @@ struct paiext_map { struct pai_userdata *save; /* Area to store non-zero counters */ enum paievt_mode mode; /* Type of event */ unsigned int active_events; /* # of PAI Extension users */ - unsigned int refcnt; + refcount_t refcnt; struct perf_event *event; /* Perf event for sampling */ struct paiext_cb *paiext_cb; /* PAI extension control block area */ }; @@ -60,14 +60,14 @@ struct paiext_mapptr { }; static struct paiext_root { /* Anchor to per CPU data */ - int refcnt; /* Overall active events */ + refcount_t refcnt; /* Overall active events */ struct paiext_mapptr __percpu *mapptr; } paiext_root; /* Free per CPU data when the last event is removed. */ static void paiext_root_free(void) { - if (!--paiext_root.refcnt) { + if (refcount_dec_and_test(&paiext_root.refcnt)) { free_percpu(paiext_root.mapptr); paiext_root.mapptr = NULL; } @@ -80,7 +80,7 @@ static void paiext_root_free(void) */ static int paiext_root_alloc(void) { - if (++paiext_root.refcnt == 1) { + if (!refcount_inc_not_zero(&paiext_root.refcnt)) { /* The memory is already zeroed. */ paiext_root.mapptr = alloc_percpu(struct paiext_mapptr); if (!paiext_root.mapptr) { @@ -91,6 +91,7 @@ static int paiext_root_alloc(void) */ return -ENOMEM; } + refcount_set(&paiext_root.refcnt, 1); } return 0; } @@ -122,7 +123,7 @@ static void paiext_event_destroy(struct perf_event *event) mutex_lock(&paiext_reserve_mutex); cpump->event = NULL; - if (!--cpump->refcnt) /* Last reference gone */ + if (refcount_dec_and_test(&cpump->refcnt)) /* Last reference gone */ paiext_free(mp); paiext_root_free(); mutex_unlock(&paiext_reserve_mutex); @@ -163,7 +164,7 @@ static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event) rc = -ENOMEM; cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); if (!cpump) - goto unlock; + goto undo; /* Allocate memory for counter area and counter extraction. * These are @@ -183,8 +184,9 @@ static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event) GFP_KERNEL); if (!cpump->save || !cpump->area || !cpump->paiext_cb) { paiext_free(mp); - goto unlock; + goto undo; } + refcount_set(&cpump->refcnt, 1); cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING; } else { @@ -195,15 +197,15 @@ static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event) if (cpump->mode == PAI_MODE_SAMPLING || (cpump->mode == PAI_MODE_COUNTING && a->sample_period)) { rc = -EBUSY; - goto unlock; + goto undo; } + refcount_inc(&cpump->refcnt); } rc = 0; cpump->event = event; - ++cpump->refcnt; -unlock: +undo: if (rc) { /* Error in allocation of event, decrement anchor. Since * the event in not created, its destroy() function is never @@ -211,6 +213,7 @@ unlock: */ paiext_root_free(); } +unlock: mutex_unlock(&paiext_reserve_mutex); /* If rc is non-zero, no increment of counter/sampler was done. */ return rc; -- cgit v1.2.3 From fbac266f095de92b93e7c91c71cdca432c2c9275 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 25 Apr 2023 13:42:11 +0200 Subject: s390: select ARCH_SUPPORTS_INT128 s390 has instructions to support 128 bit arithmetics, e.g. a 64 bit multiply instruction with a 128 bit result. Also 128 bit integer artithmetics are already used in s390 specific architecture code (see e.g. read_persistent_clock64()). Therefore select ARCH_SUPPORTS_INT128. However limit this to clang for now, since gcc generates inefficient code, which may lead to stack overflows, when compiling lib/crypto/curve25519-hacl64.c which depends on ARCH_SUPPORTS_INT128. The gcc generated functions have 6kb stack frames, compared to only 1kb of the code generated with clang. If the kernel is compiled with -Os library calls for __ashlti3(), __ashrti3(), and __lshrti3() may be generated. Similar to arm64 and riscv provide assembler implementations for these functions. Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/Kconfig | 1 + arch/s390/include/asm/asm-prototypes.h | 4 +++ arch/s390/lib/Makefile | 2 +- arch/s390/lib/tishift.S | 63 ++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 arch/s390/lib/tishift.S diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index db20c1589a98..0c1bfac08e05 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -117,6 +117,7 @@ config S390 select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_DEBUG_PAGEALLOC select ARCH_SUPPORTS_HUGETLBFS + select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && CC_IS_CLANG select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_USE_BUILTIN_BSWAP diff --git a/arch/s390/include/asm/asm-prototypes.h b/arch/s390/include/asm/asm-prototypes.h index c37eb921bfbf..a873e873e1ee 100644 --- a/arch/s390/include/asm/asm-prototypes.h +++ b/arch/s390/include/asm/asm-prototypes.h @@ -6,4 +6,8 @@ #include #include +__int128_t __ashlti3(__int128_t a, int b); +__int128_t __ashrti3(__int128_t a, int b); +__int128_t __lshrti3(__int128_t a, int b); + #endif /* _ASM_S390_PROTOTYPES_H */ diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index 580d2e3265cb..7c50eca85ca4 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -3,7 +3,7 @@ # Makefile for s390-specific library files.. # -lib-y += delay.o string.o uaccess.o find.o spinlock.o +lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o obj-y += mem.o xor.o lib-$(CONFIG_KPROBES) += probes.o lib-$(CONFIG_UPROBES) += probes.o diff --git a/arch/s390/lib/tishift.S b/arch/s390/lib/tishift.S new file mode 100644 index 000000000000..de33cf02cfd2 --- /dev/null +++ b/arch/s390/lib/tishift.S @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include +#include +#include + + .section .noinstr.text, "ax" + + GEN_BR_THUNK %r14 + +SYM_FUNC_START(__ashlti3) + lmg %r0,%r1,0(%r3) + cije %r4,0,1f + lhi %r3,64 + sr %r3,%r4 + jnh 0f + srlg %r3,%r1,0(%r3) + sllg %r0,%r0,0(%r4) + sllg %r1,%r1,0(%r4) + ogr %r0,%r3 + j 1f +0: sllg %r0,%r1,-64(%r4) + lghi %r1,0 +1: stmg %r0,%r1,0(%r2) + BR_EX %r14 +SYM_FUNC_END(__ashlti3) +EXPORT_SYMBOL(__ashlti3) + +SYM_FUNC_START(__ashrti3) + lmg %r0,%r1,0(%r3) + cije %r4,0,1f + lhi %r3,64 + sr %r3,%r4 + jnh 0f + sllg %r3,%r0,0(%r3) + srlg %r1,%r1,0(%r4) + srag %r0,%r0,0(%r4) + ogr %r1,%r3 + j 1f +0: srag %r1,%r0,-64(%r4) + srag %r0,%r0,63 +1: stmg %r0,%r1,0(%r2) + BR_EX %r14 +SYM_FUNC_END(__ashrti3) +EXPORT_SYMBOL(__ashrti3) + +SYM_FUNC_START(__lshrti3) + lmg %r0,%r1,0(%r3) + cije %r4,0,1f + lhi %r3,64 + sr %r3,%r4 + jnh 0f + sllg %r3,%r0,0(%r3) + srlg %r1,%r1,0(%r4) + srlg %r0,%r0,0(%r4) + ogr %r1,%r3 + j 1f +0: srlg %r1,%r0,-64(%r4) + lghi %r0,0 +1: stmg %r0,%r1,0(%r2) + BR_EX %r14 +SYM_FUNC_END(__lshrti3) +EXPORT_SYMBOL(__lshrti3) -- cgit v1.2.3 From 5dd4241964c86e866ae51b5c5b95cd14c5f341e6 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 22 May 2023 19:35:12 -0600 Subject: vfio/ccw: replace one-element array with flexible-array member One-element arrays are deprecated, and we are replacing them with flexible array members instead. So, replace one-element array with flexible-array member in struct vfio_ccw_parent and refactor the rest of the code accordingly. Link: https://github.com/KSPP/linux/issues/79 Link: https://github.com/KSPP/linux/issues/297 Signed-off-by: Gustavo A. R. Silva Reviewed-by: Kees Cook Reviewed-by: Eric Farman Reviewed-by: Matthew Rosato Link: https://lore.kernel.org/r/3c10549ebe1564eade68a2515bde233527376971.1684805398.git.gustavoars@kernel.org Signed-off-by: Alexander Gordeev --- drivers/s390/cio/vfio_ccw_drv.c | 3 ++- drivers/s390/cio/vfio_ccw_private.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c index ff538a086fc7..57906a9c6324 100644 --- a/drivers/s390/cio/vfio_ccw_drv.c +++ b/drivers/s390/cio/vfio_ccw_drv.c @@ -171,7 +171,8 @@ static int vfio_ccw_sch_probe(struct subchannel *sch) return -ENODEV; } - parent = kzalloc(sizeof(*parent), GFP_KERNEL); + parent = kzalloc(sizeof(*parent) + sizeof(struct mdev_type *), + GFP_KERNEL); if (!parent) return -ENOMEM; diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h index b441ae6700fd..b62bbc5c6376 100644 --- a/drivers/s390/cio/vfio_ccw_private.h +++ b/drivers/s390/cio/vfio_ccw_private.h @@ -79,7 +79,7 @@ struct vfio_ccw_parent { struct mdev_parent parent; struct mdev_type mdev_type; - struct mdev_type *mdev_types[1]; + struct mdev_type *mdev_types[]; }; /** -- cgit v1.2.3 From d933e5f41e4f6c54f4bd3e0b29ca4854fe5fa0d6 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 22 May 2023 19:35:59 -0600 Subject: vfio/ccw: use struct_size() helper Prefer struct_size() over open-coded versions. Link: https://github.com/KSPP/linux/issues/160 Signed-off-by: Gustavo A. R. Silva Reviewed-by: Kees Cook Reviewed-by: Eric Farman Reviewed-by: Matthew Rosato Link: https://lore.kernel.org/r/f657276073630e806e69726a40ad1cc85101448a.1684805398.git.gustavoars@kernel.org Signed-off-by: Alexander Gordeev --- drivers/s390/cio/vfio_ccw_drv.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c index 57906a9c6324..43601816ea4e 100644 --- a/drivers/s390/cio/vfio_ccw_drv.c +++ b/drivers/s390/cio/vfio_ccw_drv.c @@ -171,8 +171,7 @@ static int vfio_ccw_sch_probe(struct subchannel *sch) return -ENODEV; } - parent = kzalloc(sizeof(*parent) + sizeof(struct mdev_type *), - GFP_KERNEL); + parent = kzalloc(struct_size(parent, mdev_types, 1), GFP_KERNEL); if (!parent) return -ENOMEM; -- cgit v1.2.3 From 31e9ccc67ce24f82120e41fcafd841f98838ff5c Mon Sep 17 00:00:00 2001 From: Mikhail Zaslonko Date: Thu, 30 Mar 2023 09:18:01 +0200 Subject: s390/ipl: add REIPL_CLEAR flag to os_info Introduce new OS_INFO_FLAGS_ENTRY to os_info pointing to the field with bit flags. Add OS_INFO_FLAGS_ENTRY upon dump_reipl shutdown action processing and set OS_INFO_FLAG_REIPL_CLEAR flag indicating 'clear' sysfs attribute has been set on the panicked system for specified ipl type. This flag can be used to inform the dumper whether LOAD_CLEAR or LOAD_NORMAL diag308 subcode to be used for ipl after dumping the memory. Signed-off-by: Mikhail Zaslonko Acked-by: Alexander Gordeev Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/os_info.h | 7 +++++-- arch/s390/kernel/ipl.c | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/os_info.h b/arch/s390/include/asm/os_info.h index 0d1c74a7a650..a4d2e103f116 100644 --- a/arch/s390/include/asm/os_info.h +++ b/arch/s390/include/asm/os_info.h @@ -16,6 +16,9 @@ #define OS_INFO_VMCOREINFO 0 #define OS_INFO_REIPL_BLOCK 1 +#define OS_INFO_FLAGS_ENTRY 2 + +#define OS_INFO_FLAG_REIPL_CLEAR (1UL << 0) struct os_info_entry { u64 addr; @@ -30,8 +33,8 @@ struct os_info { u16 version_minor; u64 crashkernel_addr; u64 crashkernel_size; - struct os_info_entry entry[2]; - u8 reserved[4024]; + struct os_info_entry entry[3]; + u8 reserved[4004]; } __packed; void os_info_init(void); diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 43de939b7af1..d57b02fdd329 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -176,6 +176,8 @@ static bool reipl_fcp_clear; static bool reipl_ccw_clear; static bool reipl_eckd_clear; +static unsigned long os_info_flags; + static inline int __diag308(unsigned long subcode, unsigned long addr) { union register_pair r1; @@ -1939,6 +1941,20 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) struct lowcore *abs_lc; unsigned int csum; + /* + * Set REIPL_CLEAR flag in os_info flags entry indicating + * 'clear' sysfs attribute has been set on the panicked system + * for specified reipl type. + * Always set for IPL_TYPE_NSS and IPL_TYPE_UNKNOWN. + */ + if ((reipl_type == IPL_TYPE_CCW && reipl_ccw_clear) || + (reipl_type == IPL_TYPE_ECKD && reipl_eckd_clear) || + (reipl_type == IPL_TYPE_FCP && reipl_fcp_clear) || + (reipl_type == IPL_TYPE_NVME && reipl_nvme_clear) || + reipl_type == IPL_TYPE_NSS || + reipl_type == IPL_TYPE_UNKNOWN) + os_info_flags |= OS_INFO_FLAG_REIPL_CLEAR; + os_info_entry_add(OS_INFO_FLAGS_ENTRY, &os_info_flags, sizeof(os_info_flags)); csum = (__force unsigned int) csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); abs_lc = get_abs_lowcore(); -- cgit v1.2.3 From 9f70bc890ae299a6fdf83bfc99832509fd2f7494 Mon Sep 17 00:00:00 2001 From: Mikhail Zaslonko Date: Tue, 9 May 2023 17:38:15 +0200 Subject: s390/zcore: conditionally clear memory on reipl Currently zcore_reipl_write() always use DIAG308_LOAD_CLEAR to ipl the system (used by ngdump and zfcpdump to ipl the original kernel after dumping). Instead of 'always clear' check OS_INFO_FLAG_REIPL_CLEAR flag and use either LOAD_CLEAR or LOAD_NORMAL diag308 subcode accordingly. For that read os_info and check for the valid os_info flags entry in zcore_reipl_init(). Do not return error on failure (e.g. os_info checksum error) in order to continue dump processing, considering that os_info could be corrupted on the panicked system. Signed-off-by: Mikhail Zaslonko Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- drivers/s390/char/zcore.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index 599f547310f8..942c73a11ca3 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -51,6 +51,7 @@ static struct dentry *zcore_dir; static struct dentry *zcore_reipl_file; static struct dentry *zcore_hsa_file; static struct ipl_parameter_block *zcore_ipl_block; +static unsigned long os_info_flags; static DEFINE_MUTEX(hsa_buf_mutex); static char hsa_buf[PAGE_SIZE] __aligned(PAGE_SIZE); @@ -139,7 +140,13 @@ static ssize_t zcore_reipl_write(struct file *filp, const char __user *buf, { if (zcore_ipl_block) { diag308(DIAG308_SET, zcore_ipl_block); - diag308(DIAG308_LOAD_CLEAR, NULL); + if (os_info_flags & OS_INFO_FLAG_REIPL_CLEAR) + diag308(DIAG308_LOAD_CLEAR, NULL); + /* Use special diag308 subcode for CCW normal ipl */ + if (zcore_ipl_block->pb0_hdr.pbt == IPL_PBT_CCW) + diag308(DIAG308_LOAD_NORMAL_DUMP, NULL); + else + diag308(DIAG308_LOAD_NORMAL, NULL); } return count; } @@ -212,7 +219,10 @@ static int __init check_sdias(void) */ static int __init zcore_reipl_init(void) { + struct os_info_entry *entry; struct ipib_info ipib_info; + unsigned long os_info_addr; + struct os_info *os_info; int rc; rc = memcpy_hsa_kernel(&ipib_info, __LC_DUMP_REIPL, sizeof(ipib_info)); @@ -234,6 +244,35 @@ static int __init zcore_reipl_init(void) free_page((unsigned long) zcore_ipl_block); zcore_ipl_block = NULL; } + /* + * Read the bit-flags field from os_info flags entry. + * Return zero even for os_info read or entry checksum errors in order + * to continue dump processing, considering that os_info could be + * corrupted on the panicked system. + */ + os_info = (void *)__get_free_page(GFP_KERNEL); + if (!os_info) + return -ENOMEM; + rc = memcpy_hsa_kernel(&os_info_addr, __LC_OS_INFO, sizeof(os_info_addr)); + if (rc) + goto out; + if (os_info_addr < sclp.hsa_size) + rc = memcpy_hsa_kernel(os_info, os_info_addr, PAGE_SIZE); + else + rc = memcpy_real(os_info, os_info_addr, PAGE_SIZE); + if (rc || os_info_csum(os_info) != os_info->csum) + goto out; + entry = &os_info->entry[OS_INFO_FLAGS_ENTRY]; + if (entry->addr && entry->size) { + if (entry->addr < sclp.hsa_size) + rc = memcpy_hsa_kernel(&os_info_flags, entry->addr, sizeof(os_info_flags)); + else + rc = memcpy_real(&os_info_flags, entry->addr, sizeof(os_info_flags)); + if (rc || (__force u32)csum_partial(&os_info_flags, entry->size, 0) != entry->csum) + os_info_flags = 0; + } +out: + free_page((unsigned long)os_info); return 0; } -- cgit v1.2.3 From 46a29b039e2ea1199ef59c68f4313a8eebbd7d56 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Wed, 5 Apr 2023 17:29:32 +0200 Subject: s390/pkey: introduce reverse x-mas trees This patch introduces reverse x-mas trees for all local variables on all the functions in pkey. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/pkey_api.c | 72 +++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index a8def50c149b..aaf29fb70940 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -91,9 +91,9 @@ static int pkey_clr2protkey(u32 keytype, /* mask of available pckmo subfunctions */ static cpacf_mask_t pckmo_functions; - long fc; - int keysize; u8 paramblock[64]; + int keysize; + long fc; switch (keytype) { case PKEY_KEYTYPE_AES_128: @@ -146,9 +146,9 @@ static int pkey_clr2protkey(u32 keytype, */ static int pkey_skey2pkey(const u8 *key, struct pkey_protkey *pkey) { - int rc, verify; - u16 cardnr, domain; struct keytoken_header *hdr = (struct keytoken_header *)key; + u16 cardnr, domain; + int rc, verify; zcrypt_wait_api_operational(); @@ -195,9 +195,9 @@ static int pkey_skey2pkey(const u8 *key, struct pkey_protkey *pkey) static int pkey_clr2ep11key(const u8 *clrkey, size_t clrkeylen, u8 *keybuf, size_t *keybuflen) { - int i, rc; - u16 card, dom; u32 nr_apqns, *apqns = NULL; + u16 card, dom; + int i, rc; zcrypt_wait_api_operational(); @@ -229,10 +229,10 @@ out: */ static int pkey_ep11key2pkey(const u8 *key, struct pkey_protkey *pkey) { - int i, rc; - u16 card, dom; - u32 nr_apqns, *apqns = NULL; struct ep11keyblob *kb = (struct ep11keyblob *)key; + u32 nr_apqns, *apqns = NULL; + u16 card, dom; + int i, rc; zcrypt_wait_api_operational(); @@ -347,13 +347,13 @@ static int pkey_genprotkey(u32 keytype, struct pkey_protkey *protkey) */ static int pkey_verifyprotkey(const struct pkey_protkey *protkey) { - unsigned long fc; struct { u8 iv[AES_BLOCK_SIZE]; u8 key[MAXPROTKEYSIZE]; } param; u8 null_msg[AES_BLOCK_SIZE]; u8 dest_buf[AES_BLOCK_SIZE]; + unsigned long fc; unsigned int k; switch (protkey->type) { @@ -393,9 +393,9 @@ static int pkey_verifyprotkey(const struct pkey_protkey *protkey) static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, struct pkey_protkey *protkey) { - int rc = -EINVAL; - u8 *tmpbuf = NULL; struct keytoken_header *hdr = (struct keytoken_header *)key; + u8 *tmpbuf = NULL; + int rc = -EINVAL; switch (hdr->version) { case TOKVER_PROTECTED_KEY: { @@ -518,8 +518,8 @@ static int pkey_ccainttok2pkey(const u8 *key, u32 keylen, int pkey_keyblob2pkey(const u8 *key, u32 keylen, struct pkey_protkey *protkey) { - int rc; struct keytoken_header *hdr = (struct keytoken_header *)key; + int rc; if (keylen < sizeof(struct keytoken_header)) { DEBUG_ERR("%s invalid keylen %d\n", __func__, keylen); @@ -663,9 +663,9 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, enum pkey_key_type *ktype, enum pkey_key_size *ksize, u32 *flags) { - int rc; - u32 _nr_apqns, *_apqns = NULL; struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 _nr_apqns, *_apqns = NULL; + int rc; if (keylen < sizeof(struct keytoken_header)) return -EINVAL; @@ -773,8 +773,8 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, const u8 *key, size_t keylen, struct pkey_protkey *pkey) { - int i, card, dom, rc; struct keytoken_header *hdr = (struct keytoken_header *)key; + int i, card, dom, rc; /* check for at least one apqn given */ if (!apqns || !nr_apqns) @@ -847,9 +847,9 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, struct pkey_apqn *apqns, size_t *nr_apqns) { - int rc; - u32 _nr_apqns, *_apqns = NULL; struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 _nr_apqns, *_apqns = NULL; + int rc; if (keylen < sizeof(struct keytoken_header) || flags == 0) return -EINVAL; @@ -860,9 +860,9 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, (hdr->version == TOKVER_EP11_AES_WITH_HEADER || hdr->version == TOKVER_EP11_ECC_WITH_HEADER) && is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { - int minhwtype = 0, api = 0; struct ep11keyblob *kb = (struct ep11keyblob *) (key + sizeof(struct ep11kblob_header)); + int minhwtype = 0, api = 0; if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) return -EINVAL; @@ -877,8 +877,8 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, } else if (hdr->type == TOKTYPE_NON_CCA && hdr->version == TOKVER_EP11_AES && is_ep11_keyblob(key)) { - int minhwtype = 0, api = 0; struct ep11keyblob *kb = (struct ep11keyblob *)key; + int minhwtype = 0, api = 0; if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) return -EINVAL; @@ -891,8 +891,8 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, if (rc) goto out; } else if (hdr->type == TOKTYPE_CCA_INTERNAL) { - int minhwtype = ZCRYPT_CEX3C; u64 cur_mkvp = 0, old_mkvp = 0; + int minhwtype = ZCRYPT_CEX3C; if (hdr->version == TOKVER_CCA_AES) { struct secaeskeytoken *t = (struct secaeskeytoken *)key; @@ -919,8 +919,8 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, if (rc) goto out; } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { - u64 cur_mkvp = 0, old_mkvp = 0; struct eccprivkeytoken *t = (struct eccprivkeytoken *)key; + u64 cur_mkvp = 0, old_mkvp = 0; if (t->secid == 0x20) { if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) @@ -957,8 +957,8 @@ static int pkey_apqns4keytype(enum pkey_key_type ktype, u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, struct pkey_apqn *apqns, size_t *nr_apqns) { - int rc; u32 _nr_apqns, *_apqns = NULL; + int rc; zcrypt_wait_api_operational(); @@ -1023,8 +1023,8 @@ static int pkey_keyblob2pkey3(const struct pkey_apqn *apqns, size_t nr_apqns, const u8 *key, size_t keylen, u32 *protkeytype, u8 *protkey, u32 *protkeylen) { - int i, card, dom, rc; struct keytoken_header *hdr = (struct keytoken_header *)key; + int i, card, dom, rc; /* check for at least one apqn given */ if (!apqns || !nr_apqns) @@ -1303,9 +1303,9 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } case PKEY_GENSECK2: { struct pkey_genseck2 __user *ugs = (void __user *)arg; + size_t klen = KEYBLOBBUFSIZE; struct pkey_genseck2 kgs; struct pkey_apqn *apqns; - size_t klen = KEYBLOBBUFSIZE; u8 *kkey; if (copy_from_user(&kgs, ugs, sizeof(kgs))) @@ -1345,9 +1345,9 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } case PKEY_CLR2SECK2: { struct pkey_clr2seck2 __user *ucs = (void __user *)arg; + size_t klen = KEYBLOBBUFSIZE; struct pkey_clr2seck2 kcs; struct pkey_apqn *apqns; - size_t klen = KEYBLOBBUFSIZE; u8 *kkey; if (copy_from_user(&kcs, ucs, sizeof(kcs))) @@ -1409,8 +1409,8 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } case PKEY_KBLOB2PROTK2: { struct pkey_kblob2pkey2 __user *utp = (void __user *)arg; - struct pkey_kblob2pkey2 ktp; struct pkey_apqn *apqns = NULL; + struct pkey_kblob2pkey2 ktp; u8 *kkey; if (copy_from_user(&ktp, utp, sizeof(ktp))) @@ -1437,8 +1437,8 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } case PKEY_APQNS4K: { struct pkey_apqns4key __user *uak = (void __user *)arg; - struct pkey_apqns4key kak; struct pkey_apqn *apqns = NULL; + struct pkey_apqns4key kak; size_t nr_apqns, len; u8 *kkey; @@ -1486,8 +1486,8 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } case PKEY_APQNS4KT: { struct pkey_apqns4keytype __user *uat = (void __user *)arg; - struct pkey_apqns4keytype kat; struct pkey_apqn *apqns = NULL; + struct pkey_apqns4keytype kat; size_t nr_apqns, len; if (copy_from_user(&kat, uat, sizeof(kat))) @@ -1528,9 +1528,9 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } case PKEY_KBLOB2PROTK3: { struct pkey_kblob2pkey3 __user *utp = (void __user *)arg; - struct pkey_kblob2pkey3 ktp; - struct pkey_apqn *apqns = NULL; u32 protkeylen = PROTKEYBLOBBUFSIZE; + struct pkey_apqn *apqns = NULL; + struct pkey_kblob2pkey3 ktp; u8 *kkey, *protkey; if (copy_from_user(&ktp, utp, sizeof(ktp))) @@ -1717,8 +1717,8 @@ static struct attribute_group protkey_attr_group = { static ssize_t pkey_ccadata_aes_attr_read(u32 keytype, bool is_xts, char *buf, loff_t off, size_t count) { - int rc; struct pkey_seckey *seckey = (struct pkey_seckey *)buf; + int rc; if (off != 0 || count < sizeof(struct secaeskeytoken)) return -EINVAL; @@ -1824,9 +1824,9 @@ static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, bool is_xts, char *buf, loff_t off, size_t count) { - int i, rc, card, dom; - u32 nr_apqns, *apqns = NULL; size_t keysize = CCACIPHERTOKENSIZE; + u32 nr_apqns, *apqns = NULL; + int i, rc, card, dom; if (off != 0 || count < CCACIPHERTOKENSIZE) return -EINVAL; @@ -1947,9 +1947,9 @@ static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, bool is_xts, char *buf, loff_t off, size_t count) { - int i, rc, card, dom; - u32 nr_apqns, *apqns = NULL; size_t keysize = MAXEP11AESKEYBLOBSIZE; + u32 nr_apqns, *apqns = NULL; + int i, rc, card, dom; if (off != 0 || count < MAXEP11AESKEYBLOBSIZE) return -EINVAL; -- cgit v1.2.3 From f370f45c6475ad0058277ae111f28fb32f58aa46 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Sat, 1 Apr 2023 14:32:08 +0200 Subject: s390/pkey: do not use struct pkey_protkey This is an internal rework of the pkey code to not use the struct pkey_protkey internal any more. This struct has a hard coded protected key buffer with MAXPROTKEYSIZE = 64 bytes. However, with support for ECC protected key, this limit is too short and thus this patch reworks all the internal code to use the triple u8 *protkey, u32 protkeylen, u32 protkeytype instead. So the ioctl which still has to deal with this struct coming from userspace and/or provided to userspace invoke all the internal functions now with the triple instead of passing a pointer to struct pkey_protkey. Also the struct pkey_clrkey has been internally replaced in a similar way. This struct also has a hard coded clear key buffer of MAXCLRKEYSIZE = 32 bytes and thus is not usable with e.g. ECC clear key material. This is a transparent rework for userspace applications using the pkey API. The internal kernel API used by the PAES crypto ciphers has been adapted to this change to make it possible to provide ECC protected keys via this interface in the future. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Alexander Gordeev --- arch/s390/crypto/paes_s390.c | 9 +- arch/s390/include/asm/pkey.h | 4 +- drivers/s390/crypto/pkey_api.c | 195 ++++++++++++++++++++++++----------------- 3 files changed, 125 insertions(+), 83 deletions(-) diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 29dc827e0fe8..d29a9d908797 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -5,7 +5,7 @@ * s390 implementation of the AES Cipher Algorithm with protected keys. * * s390 Version: - * Copyright IBM Corp. 2017,2020 + * Copyright IBM Corp. 2017, 2023 * Author(s): Martin Schwidefsky * Harald Freudenberger */ @@ -132,7 +132,8 @@ static inline int __paes_keyblob2pkey(struct key_blob *kb, if (i > 0 && ret == -EAGAIN && in_task()) if (msleep_interruptible(1000)) return -EINTR; - ret = pkey_keyblob2pkey(kb->key, kb->keylen, pk); + ret = pkey_keyblob2pkey(kb->key, kb->keylen, + pk->protkey, &pk->len, &pk->type); if (ret == 0) break; } @@ -145,6 +146,7 @@ static inline int __paes_convert_key(struct s390_paes_ctx *ctx) int ret; struct pkey_protkey pkey; + pkey.len = sizeof(pkey.protkey); ret = __paes_keyblob2pkey(&ctx->kb, &pkey); if (ret) return ret; @@ -414,6 +416,9 @@ static inline int __xts_paes_convert_key(struct s390_pxts_ctx *ctx) { struct pkey_protkey pkey0, pkey1; + pkey0.len = sizeof(pkey0.protkey); + pkey1.len = sizeof(pkey1.protkey); + if (__paes_keyblob2pkey(&ctx->kb[0], &pkey0) || __paes_keyblob2pkey(&ctx->kb[1], &pkey1)) return -EINVAL; diff --git a/arch/s390/include/asm/pkey.h b/arch/s390/include/asm/pkey.h index dd3d20c332ac..47d80a7451a6 100644 --- a/arch/s390/include/asm/pkey.h +++ b/arch/s390/include/asm/pkey.h @@ -2,7 +2,7 @@ /* * Kernelspace interface to the pkey device driver * - * Copyright IBM Corp. 2016,2019 + * Copyright IBM Corp. 2016, 2023 * * Author: Harald Freudenberger * @@ -23,6 +23,6 @@ * @return 0 on success, negative errno value on failure */ int pkey_keyblob2pkey(const u8 *key, u32 keylen, - struct pkey_protkey *protkey); + u8 *protkey, u32 *protkeylen, u32 *protkeytype); #endif /* _KAPI_PKEY_H */ diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index aaf29fb70940..9d2af01204ea 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -34,6 +34,7 @@ MODULE_DESCRIPTION("s390 protected key interface"); #define KEYBLOBBUFSIZE 8192 /* key buffer size used for internal processing */ #define PROTKEYBLOBBUFSIZE 256 /* protected key buffer size used internal */ #define MAXAPQNSINLIST 64 /* max 64 apqns within a apqn list */ +#define AES_WK_VP_SIZE 32 /* Size of WK VP block appended to a prot key */ /* * debug feature data and functions @@ -82,11 +83,10 @@ struct clearaeskeytoken { } __packed; /* - * Create a protected key from a clear key value. + * Create a protected key from a clear key value via PCKMO instruction. */ -static int pkey_clr2protkey(u32 keytype, - const struct pkey_clrkey *clrkey, - struct pkey_protkey *protkey) +static int pkey_clr2protkey(u32 keytype, const u8 *clrkey, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { /* mask of available pckmo subfunctions */ static cpacf_mask_t pckmo_functions; @@ -109,11 +109,17 @@ static int pkey_clr2protkey(u32 keytype, fc = CPACF_PCKMO_ENC_AES_256_KEY; break; default: - DEBUG_ERR("%s unknown/unsupported keytype %d\n", + DEBUG_ERR("%s unknown/unsupported keytype %u\n", __func__, keytype); return -EINVAL; } + if (*protkeylen < keysize + AES_WK_VP_SIZE) { + DEBUG_ERR("%s prot key buffer size too small: %u < %d\n", + __func__, *protkeylen, keysize + AES_WK_VP_SIZE); + return -EINVAL; + } + /* Did we already check for PCKMO ? */ if (!pckmo_functions.bytes[0]) { /* no, so check now */ @@ -128,15 +134,15 @@ static int pkey_clr2protkey(u32 keytype, /* prepare param block */ memset(paramblock, 0, sizeof(paramblock)); - memcpy(paramblock, clrkey->clrkey, keysize); + memcpy(paramblock, clrkey, keysize); /* call the pckmo instruction */ cpacf_pckmo(fc, paramblock); - /* copy created protected key */ - protkey->type = keytype; - protkey->len = keysize + 32; - memcpy(protkey->protkey, paramblock, keysize + 32); + /* copy created protected key to key buffer including the wkvp block */ + *protkeylen = keysize + AES_WK_VP_SIZE; + memcpy(protkey, paramblock, *protkeylen); + *protkeytype = keytype; return 0; } @@ -144,7 +150,8 @@ static int pkey_clr2protkey(u32 keytype, /* * Find card and transform secure key into protected key. */ -static int pkey_skey2pkey(const u8 *key, struct pkey_protkey *pkey) +static int pkey_skey2pkey(const u8 *key, u8 *protkey, + u32 *protkeylen, u32 *protkeytype) { struct keytoken_header *hdr = (struct keytoken_header *)key; u16 cardnr, domain; @@ -167,14 +174,13 @@ static int pkey_skey2pkey(const u8 *key, struct pkey_protkey *pkey) continue; switch (hdr->version) { case TOKVER_CCA_AES: - rc = cca_sec2protkey(cardnr, domain, - key, pkey->protkey, - &pkey->len, &pkey->type); + rc = cca_sec2protkey(cardnr, domain, key, + protkey, protkeylen, protkeytype); break; case TOKVER_CCA_VLSC: - rc = cca_cipher2protkey(cardnr, domain, - key, pkey->protkey, - &pkey->len, &pkey->type); + rc = cca_cipher2protkey(cardnr, domain, key, + protkey, protkeylen, + protkeytype); break; default: return -EINVAL; @@ -227,7 +233,8 @@ out: /* * Find card and transform EP11 secure key into protected key. */ -static int pkey_ep11key2pkey(const u8 *key, struct pkey_protkey *pkey) +static int pkey_ep11key2pkey(const u8 *key, u8 *protkey, + u32 *protkeylen, u32 *protkeytype) { struct ep11keyblob *kb = (struct ep11keyblob *)key; u32 nr_apqns, *apqns = NULL; @@ -246,9 +253,8 @@ static int pkey_ep11key2pkey(const u8 *key, struct pkey_protkey *pkey) for (rc = -ENODEV, i = 0; i < nr_apqns; i++) { card = apqns[i] >> 16; dom = apqns[i] & 0xFFFF; - pkey->len = sizeof(pkey->protkey); rc = ep11_kblob2protkey(card, dom, key, kb->head.len, - pkey->protkey, &pkey->len, &pkey->type); + protkey, protkeylen, protkeytype); if (rc == 0) break; } @@ -306,9 +312,10 @@ out: /* * Generate a random protected key */ -static int pkey_genprotkey(u32 keytype, struct pkey_protkey *protkey) +static int pkey_genprotkey(u32 keytype, u8 *protkey, + u32 *protkeylen, u32 *protkeytype) { - struct pkey_clrkey clrkey; + u8 clrkey[32]; int keysize; int rc; @@ -329,15 +336,16 @@ static int pkey_genprotkey(u32 keytype, struct pkey_protkey *protkey) } /* generate a dummy random clear key */ - get_random_bytes(clrkey.clrkey, keysize); + get_random_bytes(clrkey, keysize); /* convert it to a dummy protected key */ - rc = pkey_clr2protkey(keytype, &clrkey, protkey); + rc = pkey_clr2protkey(keytype, clrkey, + protkey, protkeylen, protkeytype); if (rc) return rc; /* replace the key part of the protected key with random bytes */ - get_random_bytes(protkey->protkey, keysize); + get_random_bytes(protkey, keysize); return 0; } @@ -345,7 +353,8 @@ static int pkey_genprotkey(u32 keytype, struct pkey_protkey *protkey) /* * Verify if a protected key is still valid */ -static int pkey_verifyprotkey(const struct pkey_protkey *protkey) +static int pkey_verifyprotkey(const u8 *protkey, u32 protkeylen, + u32 protkeytype) { struct { u8 iv[AES_BLOCK_SIZE]; @@ -353,29 +362,37 @@ static int pkey_verifyprotkey(const struct pkey_protkey *protkey) } param; u8 null_msg[AES_BLOCK_SIZE]; u8 dest_buf[AES_BLOCK_SIZE]; + unsigned int k, pkeylen; unsigned long fc; - unsigned int k; - switch (protkey->type) { + switch (protkeytype) { case PKEY_KEYTYPE_AES_128: + pkeylen = 16 + AES_WK_VP_SIZE; fc = CPACF_KMC_PAES_128; break; case PKEY_KEYTYPE_AES_192: + pkeylen = 24 + AES_WK_VP_SIZE; fc = CPACF_KMC_PAES_192; break; case PKEY_KEYTYPE_AES_256: + pkeylen = 32 + AES_WK_VP_SIZE; fc = CPACF_KMC_PAES_256; break; default: - DEBUG_ERR("%s unknown/unsupported keytype %d\n", __func__, - protkey->type); + DEBUG_ERR("%s unknown/unsupported keytype %u\n", __func__, + protkeytype); + return -EINVAL; + } + if (protkeylen != pkeylen) { + DEBUG_ERR("%s invalid protected key size %u for keytype %u\n", + __func__, protkeylen, protkeytype); return -EINVAL; } memset(null_msg, 0, sizeof(null_msg)); memset(param.iv, 0, sizeof(param.iv)); - memcpy(param.key, protkey->protkey, sizeof(param.key)); + memcpy(param.key, protkey, protkeylen); k = cpacf_kmc(fc | CPACF_ENCRYPT, ¶m, null_msg, dest_buf, sizeof(null_msg)); @@ -391,7 +408,7 @@ static int pkey_verifyprotkey(const struct pkey_protkey *protkey) * Transform a non-CCA key token into a protected key */ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, - struct pkey_protkey *protkey) + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { struct keytoken_header *hdr = (struct keytoken_header *)key; u8 *tmpbuf = NULL; @@ -404,11 +421,12 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, if (keylen != sizeof(struct protaeskeytoken)) goto out; t = (struct protaeskeytoken *)key; - protkey->len = t->len; - protkey->type = t->keytype; - memcpy(protkey->protkey, t->protkey, - sizeof(protkey->protkey)); - rc = pkey_verifyprotkey(protkey); + rc = pkey_verifyprotkey(t->protkey, t->len, t->keytype); + if (rc) + goto out; + memcpy(protkey, t->protkey, t->len); + *protkeylen = t->len; + *protkeytype = t->keytype; break; } case TOKVER_CLEAR_KEY: { @@ -438,7 +456,8 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, goto out; } /* try direct way with the PCKMO instruction */ - rc = pkey_clr2protkey(t->keytype, &ckey, protkey); + rc = pkey_clr2protkey(t->keytype, ckey.clrkey, + protkey, protkeylen, protkeytype); if (rc == 0) break; /* PCKMO failed, so try the CCA secure key way */ @@ -446,14 +465,16 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, rc = cca_clr2seckey(0xFFFF, 0xFFFF, t->keytype, ckey.clrkey, tmpbuf); if (rc == 0) - rc = pkey_skey2pkey(tmpbuf, protkey); + rc = pkey_skey2pkey(tmpbuf, + protkey, protkeylen, protkeytype); if (rc == 0) break; /* if the CCA way also failed, let's try via EP11 */ rc = pkey_clr2ep11key(ckey.clrkey, t->len, tmpbuf, &tmpbuflen); if (rc == 0) - rc = pkey_ep11key2pkey(tmpbuf, protkey); + rc = pkey_ep11key2pkey(tmpbuf, + protkey, protkeylen, protkeytype); /* now we should really have an protected key */ DEBUG_ERR("%s unable to build protected key from clear", __func__); @@ -464,7 +485,8 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, rc = ep11_check_aes_key(debug_info, 3, key, keylen, 1); if (rc) goto out; - rc = pkey_ep11key2pkey(key, protkey); + rc = pkey_ep11key2pkey(key, + protkey, protkeylen, protkeytype); break; } case TOKVER_EP11_AES_WITH_HEADER: @@ -473,7 +495,7 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, if (rc) goto out; rc = pkey_ep11key2pkey(key + sizeof(struct ep11kblob_header), - protkey); + protkey, protkeylen, protkeytype); break; default: DEBUG_ERR("%s unknown/unsupported non-CCA token version %d\n", @@ -490,7 +512,7 @@ out: * Transform a CCA internal key token into a protected key */ static int pkey_ccainttok2pkey(const u8 *key, u32 keylen, - struct pkey_protkey *protkey) + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { struct keytoken_header *hdr = (struct keytoken_header *)key; @@ -509,14 +531,14 @@ static int pkey_ccainttok2pkey(const u8 *key, u32 keylen, return -EINVAL; } - return pkey_skey2pkey(key, protkey); + return pkey_skey2pkey(key, protkey, protkeylen, protkeytype); } /* * Transform a key blob (of any type) into a protected key */ int pkey_keyblob2pkey(const u8 *key, u32 keylen, - struct pkey_protkey *protkey) + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { struct keytoken_header *hdr = (struct keytoken_header *)key; int rc; @@ -528,10 +550,12 @@ int pkey_keyblob2pkey(const u8 *key, u32 keylen, switch (hdr->type) { case TOKTYPE_NON_CCA: - rc = pkey_nonccatok2pkey(key, keylen, protkey); + rc = pkey_nonccatok2pkey(key, keylen, + protkey, protkeylen, protkeytype); break; case TOKTYPE_CCA_INTERNAL: - rc = pkey_ccainttok2pkey(key, keylen, protkey); + rc = pkey_ccainttok2pkey(key, keylen, + protkey, protkeylen, protkeytype); break; default: DEBUG_ERR("%s unknown/unsupported blob type %d\n", @@ -771,7 +795,7 @@ out: static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, const u8 *key, size_t keylen, - struct pkey_protkey *pkey) + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { struct keytoken_header *hdr = (struct keytoken_header *)key; int i, card, dom, rc; @@ -806,7 +830,9 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, if (ep11_check_aes_key(debug_info, 3, key, keylen, 1)) return -EINVAL; } else { - return pkey_nonccatok2pkey(key, keylen, pkey); + return pkey_nonccatok2pkey(key, keylen, + protkey, protkeylen, + protkeytype); } } else { DEBUG_ERR("%s unknown/unsupported blob type %d\n", @@ -822,20 +848,20 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, dom = apqns[i].domain; if (hdr->type == TOKTYPE_CCA_INTERNAL && hdr->version == TOKVER_CCA_AES) { - rc = cca_sec2protkey(card, dom, key, pkey->protkey, - &pkey->len, &pkey->type); + rc = cca_sec2protkey(card, dom, key, + protkey, protkeylen, protkeytype); } else if (hdr->type == TOKTYPE_CCA_INTERNAL && hdr->version == TOKVER_CCA_VLSC) { - rc = cca_cipher2protkey(card, dom, key, pkey->protkey, - &pkey->len, &pkey->type); + rc = cca_cipher2protkey(card, dom, key, + protkey, protkeylen, + protkeytype); } else { /* EP11 AES secure key blob */ struct ep11keyblob *kb = (struct ep11keyblob *)key; - pkey->len = sizeof(pkey->protkey); rc = ep11_kblob2protkey(card, dom, key, kb->head.len, - pkey->protkey, &pkey->len, - &pkey->type); + protkey, protkeylen, + protkeytype); } if (rc == 0) break; @@ -1020,8 +1046,8 @@ out: } static int pkey_keyblob2pkey3(const struct pkey_apqn *apqns, size_t nr_apqns, - const u8 *key, size_t keylen, u32 *protkeytype, - u8 *protkey, u32 *protkeylen) + const u8 *key, size_t keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { struct keytoken_header *hdr = (struct keytoken_header *)key; int i, card, dom, rc; @@ -1076,15 +1102,8 @@ static int pkey_keyblob2pkey3(const struct pkey_apqn *apqns, size_t nr_apqns, if (cca_check_sececckeytoken(debug_info, 3, key, keylen, 1)) return -EINVAL; } else if (hdr->type == TOKTYPE_NON_CCA) { - struct pkey_protkey pkey; - - rc = pkey_nonccatok2pkey(key, keylen, &pkey); - if (rc) - return rc; - memcpy(protkey, pkey.protkey, pkey.len); - *protkeylen = pkey.len; - *protkeytype = pkey.type; - return 0; + return pkey_nonccatok2pkey(key, keylen, + protkey, protkeylen, protkeytype); } else { DEBUG_ERR("%s unknown/unsupported blob type %d\n", __func__, hdr->type); @@ -1187,6 +1206,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, if (copy_from_user(&ksp, usp, sizeof(ksp))) return -EFAULT; + ksp.protkey.len = sizeof(ksp.protkey.protkey); rc = cca_sec2protkey(ksp.cardnr, ksp.domain, ksp.seckey.seckey, ksp.protkey.protkey, &ksp.protkey.len, &ksp.protkey.type); @@ -1203,8 +1223,10 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, if (copy_from_user(&kcp, ucp, sizeof(kcp))) return -EFAULT; - rc = pkey_clr2protkey(kcp.keytype, - &kcp.clrkey, &kcp.protkey); + kcp.protkey.len = sizeof(kcp.protkey.protkey); + rc = pkey_clr2protkey(kcp.keytype, kcp.clrkey.clrkey, + kcp.protkey.protkey, + &kcp.protkey.len, &kcp.protkey.type); DEBUG_DBG("%s pkey_clr2protkey()=%d\n", __func__, rc); if (rc) break; @@ -1234,7 +1256,9 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, if (copy_from_user(&ksp, usp, sizeof(ksp))) return -EFAULT; - rc = pkey_skey2pkey(ksp.seckey.seckey, &ksp.protkey); + ksp.protkey.len = sizeof(ksp.protkey.protkey); + rc = pkey_skey2pkey(ksp.seckey.seckey, ksp.protkey.protkey, + &ksp.protkey.len, &ksp.protkey.type); DEBUG_DBG("%s pkey_skey2pkey()=%d\n", __func__, rc); if (rc) break; @@ -1263,7 +1287,9 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, if (copy_from_user(&kgp, ugp, sizeof(kgp))) return -EFAULT; - rc = pkey_genprotkey(kgp.keytype, &kgp.protkey); + kgp.protkey.len = sizeof(kgp.protkey.protkey); + rc = pkey_genprotkey(kgp.keytype, kgp.protkey.protkey, + &kgp.protkey.len, &kgp.protkey.type); DEBUG_DBG("%s pkey_genprotkey()=%d\n", __func__, rc); if (rc) break; @@ -1277,7 +1303,8 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, if (copy_from_user(&kvp, uvp, sizeof(kvp))) return -EFAULT; - rc = pkey_verifyprotkey(&kvp.protkey); + rc = pkey_verifyprotkey(kvp.protkey.protkey, + kvp.protkey.len, kvp.protkey.type); DEBUG_DBG("%s pkey_verifyprotkey()=%d\n", __func__, rc); break; } @@ -1291,7 +1318,9 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, kkey = _copy_key_from_user(ktp.key, ktp.keylen); if (IS_ERR(kkey)) return PTR_ERR(kkey); - rc = pkey_keyblob2pkey(kkey, ktp.keylen, &ktp.protkey); + ktp.protkey.len = sizeof(ktp.protkey.protkey); + rc = pkey_keyblob2pkey(kkey, ktp.keylen, ktp.protkey.protkey, + &ktp.protkey.len, &ktp.protkey.type); DEBUG_DBG("%s pkey_keyblob2pkey()=%d\n", __func__, rc); memzero_explicit(kkey, ktp.keylen); kfree(kkey); @@ -1423,8 +1452,11 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, kfree(apqns); return PTR_ERR(kkey); } + ktp.protkey.len = sizeof(ktp.protkey.protkey); rc = pkey_keyblob2pkey2(apqns, ktp.apqn_entries, - kkey, ktp.keylen, &ktp.protkey); + kkey, ktp.keylen, + ktp.protkey.protkey, &ktp.protkey.len, + &ktp.protkey.type); DEBUG_DBG("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); kfree(apqns); memzero_explicit(kkey, ktp.keylen); @@ -1549,9 +1581,9 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, kfree(kkey); return -ENOMEM; } - rc = pkey_keyblob2pkey3(apqns, ktp.apqn_entries, kkey, - ktp.keylen, &ktp.pkeytype, - protkey, &protkeylen); + rc = pkey_keyblob2pkey3(apqns, ktp.apqn_entries, + kkey, ktp.keylen, + protkey, &protkeylen, &ktp.pkeytype); DEBUG_DBG("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); kfree(apqns); memzero_explicit(kkey, ktp.keylen); @@ -1612,7 +1644,9 @@ static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, protkeytoken.version = TOKVER_PROTECTED_KEY; protkeytoken.keytype = keytype; - rc = pkey_genprotkey(protkeytoken.keytype, &protkey); + protkey.len = sizeof(protkey.protkey); + rc = pkey_genprotkey(protkeytoken.keytype, + protkey.protkey, &protkey.len, &protkey.type); if (rc) return rc; @@ -1622,7 +1656,10 @@ static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, memcpy(buf, &protkeytoken, sizeof(protkeytoken)); if (is_xts) { - rc = pkey_genprotkey(protkeytoken.keytype, &protkey); + /* xts needs a second protected key, reuse protkey struct */ + protkey.len = sizeof(protkey.protkey); + rc = pkey_genprotkey(protkeytoken.keytype, + protkey.protkey, &protkey.len, &protkey.type); if (rc) return rc; -- cgit v1.2.3 From 9e436c195e2d6d3a0db6921e14ef2c85e559ae5b Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Sat, 1 Apr 2023 18:26:06 +0200 Subject: s390/pkey: add support for ecc clear key Add support for a new 'non CCA clear key token' with these ECC clear keys supported: - ECC P256 - ECC P384 - ECC P521 - ECC ED25519 - ECC ED448 This makes it possible to derive a protected key from this ECC clear key input via PKEY_KBLOB2PROTK3 ioctl. As of now the only way to derive protected keys from these clear key tokens is via PCKMO instruction. For AES keys an alternate path via creating a secure key from the clear key and then derive a protected key from the secure key exists. This alternate path is not implemented for ECC keys as it would require to rearrange and maybe recalculate the clear key material for input to derive an CCA or EP11 ECC secure key. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/cpacf.h | 7 +- arch/s390/include/uapi/asm/pkey.h | 15 ++- drivers/s390/crypto/pkey_api.c | 260 ++++++++++++++++++++++++++++---------- 3 files changed, 208 insertions(+), 74 deletions(-) diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index 646b12981f20..b378e2b57ad8 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -2,7 +2,7 @@ /* * CP Assist for Cryptographic Functions (CPACF) * - * Copyright IBM Corp. 2003, 2017 + * Copyright IBM Corp. 2003, 2023 * Author(s): Thomas Spatzier * Jan Glauber * Harald Freudenberger (freude@de.ibm.com) @@ -132,6 +132,11 @@ #define CPACF_PCKMO_ENC_AES_128_KEY 0x12 #define CPACF_PCKMO_ENC_AES_192_KEY 0x13 #define CPACF_PCKMO_ENC_AES_256_KEY 0x14 +#define CPACF_PCKMO_ENC_ECC_P256_KEY 0x20 +#define CPACF_PCKMO_ENC_ECC_P384_KEY 0x21 +#define CPACF_PCKMO_ENC_ECC_P521_KEY 0x22 +#define CPACF_PCKMO_ENC_ECC_ED25519_KEY 0x28 +#define CPACF_PCKMO_ENC_ECC_ED448_KEY 0x29 /* * Function codes for the PRNO (PERFORM RANDOM NUMBER OPERATION) diff --git a/arch/s390/include/uapi/asm/pkey.h b/arch/s390/include/uapi/asm/pkey.h index 924b876f992c..f7bae1c63bd6 100644 --- a/arch/s390/include/uapi/asm/pkey.h +++ b/arch/s390/include/uapi/asm/pkey.h @@ -2,7 +2,7 @@ /* * Userspace interface to the pkey device driver * - * Copyright IBM Corp. 2017, 2019 + * Copyright IBM Corp. 2017, 2023 * * Author: Harald Freudenberger * @@ -32,10 +32,15 @@ #define MINKEYBLOBSIZE SECKEYBLOBSIZE /* defines for the type field within the pkey_protkey struct */ -#define PKEY_KEYTYPE_AES_128 1 -#define PKEY_KEYTYPE_AES_192 2 -#define PKEY_KEYTYPE_AES_256 3 -#define PKEY_KEYTYPE_ECC 4 +#define PKEY_KEYTYPE_AES_128 1 +#define PKEY_KEYTYPE_AES_192 2 +#define PKEY_KEYTYPE_AES_256 3 +#define PKEY_KEYTYPE_ECC 4 +#define PKEY_KEYTYPE_ECC_P256 5 +#define PKEY_KEYTYPE_ECC_P384 6 +#define PKEY_KEYTYPE_ECC_P521 7 +#define PKEY_KEYTYPE_ECC_ED25519 8 +#define PKEY_KEYTYPE_ECC_ED448 9 /* the newer ioctls use a pkey_key_type enum for type information */ enum pkey_key_type { diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 9d2af01204ea..e58bfd225323 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -2,7 +2,8 @@ /* * pkey device driver * - * Copyright IBM Corp. 2017,2019 + * Copyright IBM Corp. 2017, 2023 + * * Author(s): Harald Freudenberger */ @@ -32,6 +33,7 @@ MODULE_AUTHOR("IBM Corporation"); MODULE_DESCRIPTION("s390 protected key interface"); #define KEYBLOBBUFSIZE 8192 /* key buffer size used for internal processing */ +#define MINKEYBLOBBUFSIZE (sizeof(struct keytoken_header)) #define PROTKEYBLOBBUFSIZE 256 /* protected key buffer size used internal */ #define MAXAPQNSINLIST 64 /* max 64 apqns within a apqn list */ #define AES_WK_VP_SIZE 32 /* Size of WK VP block appended to a prot key */ @@ -72,16 +74,31 @@ struct protaeskeytoken { } __packed; /* inside view of a clear key token (type 0x00 version 0x02) */ -struct clearaeskeytoken { - u8 type; /* 0x00 for PAES specific key tokens */ +struct clearkeytoken { + u8 type; /* 0x00 for PAES specific key tokens */ u8 res0[3]; - u8 version; /* 0x02 for clear AES key token */ + u8 version; /* 0x02 for clear key token */ u8 res1[3]; - u32 keytype; /* key type, one of the PKEY_KEYTYPE values */ - u32 len; /* bytes actually stored in clearkey[] */ + u32 keytype; /* key type, one of the PKEY_KEYTYPE_* values */ + u32 len; /* bytes actually stored in clearkey[] */ u8 clearkey[]; /* clear key value */ } __packed; +/* helper function which translates the PKEY_KEYTYPE_AES_* to their keysize */ +static inline u32 pkey_keytype_aes_to_size(u32 keytype) +{ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + return 16; + case PKEY_KEYTYPE_AES_192: + return 24; + case PKEY_KEYTYPE_AES_256: + return 32; + default: + return 0; + } +} + /* * Create a protected key from a clear key value via PCKMO instruction. */ @@ -91,23 +108,60 @@ static int pkey_clr2protkey(u32 keytype, const u8 *clrkey, /* mask of available pckmo subfunctions */ static cpacf_mask_t pckmo_functions; - u8 paramblock[64]; + u8 paramblock[112]; + u32 pkeytype; int keysize; long fc; switch (keytype) { case PKEY_KEYTYPE_AES_128: + /* 16 byte key, 32 byte aes wkvp, total 48 bytes */ keysize = 16; + pkeytype = keytype; fc = CPACF_PCKMO_ENC_AES_128_KEY; break; case PKEY_KEYTYPE_AES_192: + /* 24 byte key, 32 byte aes wkvp, total 56 bytes */ keysize = 24; + pkeytype = keytype; fc = CPACF_PCKMO_ENC_AES_192_KEY; break; case PKEY_KEYTYPE_AES_256: + /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ keysize = 32; + pkeytype = keytype; fc = CPACF_PCKMO_ENC_AES_256_KEY; break; + case PKEY_KEYTYPE_ECC_P256: + /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ + keysize = 32; + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_P256_KEY; + break; + case PKEY_KEYTYPE_ECC_P384: + /* 48 byte key, 32 byte aes wkvp, total 80 bytes */ + keysize = 48; + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_P384_KEY; + break; + case PKEY_KEYTYPE_ECC_P521: + /* 80 byte key, 32 byte aes wkvp, total 112 bytes */ + keysize = 80; + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_P521_KEY; + break; + case PKEY_KEYTYPE_ECC_ED25519: + /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ + keysize = 32; + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_ED25519_KEY; + break; + case PKEY_KEYTYPE_ECC_ED448: + /* 64 byte key, 32 byte aes wkvp, total 96 bytes */ + keysize = 64; + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_ED448_KEY; + break; default: DEBUG_ERR("%s unknown/unsupported keytype %u\n", __func__, keytype); @@ -142,7 +196,7 @@ static int pkey_clr2protkey(u32 keytype, const u8 *clrkey, /* copy created protected key to key buffer including the wkvp block */ *protkeylen = keysize + AES_WK_VP_SIZE; memcpy(protkey, paramblock, *protkeylen); - *protkeytype = keytype; + *protkeytype = pkeytype; return 0; } @@ -319,17 +373,8 @@ static int pkey_genprotkey(u32 keytype, u8 *protkey, int keysize; int rc; - switch (keytype) { - case PKEY_KEYTYPE_AES_128: - keysize = 16; - break; - case PKEY_KEYTYPE_AES_192: - keysize = 24; - break; - case PKEY_KEYTYPE_AES_256: - keysize = 32; - break; - default: + keysize = pkey_keytype_aes_to_size(keytype); + if (!keysize) { DEBUG_ERR("%s unknown/unsupported keytype %d\n", __func__, keytype); return -EINVAL; @@ -404,6 +449,111 @@ static int pkey_verifyprotkey(const u8 *protkey, u32 protkeylen, return 0; } +/* Helper for pkey_nonccatok2pkey, handles aes clear key token */ +static int nonccatokaes2pkey(const struct clearkeytoken *t, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + size_t tmpbuflen = max_t(size_t, SECKEYBLOBSIZE, MAXEP11AESKEYBLOBSIZE); + u8 *tmpbuf = NULL; + u32 keysize; + int rc; + + keysize = pkey_keytype_aes_to_size(t->keytype); + if (!keysize) { + DEBUG_ERR("%s unknown/unsupported keytype %u\n", + __func__, t->keytype); + return -EINVAL; + } + if (t->len != keysize) { + DEBUG_ERR("%s non clear key aes token: invalid key len %u\n", + __func__, t->len); + return -EINVAL; + } + + /* try direct way with the PCKMO instruction */ + rc = pkey_clr2protkey(t->keytype, t->clearkey, + protkey, protkeylen, protkeytype); + if (!rc) + goto out; + + /* PCKMO failed, so try the CCA secure key way */ + tmpbuf = kmalloc(tmpbuflen, GFP_ATOMIC); + if (!tmpbuf) + return -ENOMEM; + zcrypt_wait_api_operational(); + rc = cca_clr2seckey(0xFFFF, 0xFFFF, t->keytype, t->clearkey, tmpbuf); + if (rc) + goto try_via_ep11; + rc = pkey_skey2pkey(tmpbuf, + protkey, protkeylen, protkeytype); + if (!rc) + goto out; + +try_via_ep11: + /* if the CCA way also failed, let's try via EP11 */ + rc = pkey_clr2ep11key(t->clearkey, t->len, + tmpbuf, &tmpbuflen); + if (rc) + goto failure; + rc = pkey_ep11key2pkey(tmpbuf, + protkey, protkeylen, protkeytype); + if (!rc) + goto out; + +failure: + DEBUG_ERR("%s unable to build protected key from clear", __func__); + +out: + kfree(tmpbuf); + return rc; +} + +/* Helper for pkey_nonccatok2pkey, handles ecc clear key token */ +static int nonccatokecc2pkey(const struct clearkeytoken *t, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + u32 keylen; + int rc; + + switch (t->keytype) { + case PKEY_KEYTYPE_ECC_P256: + keylen = 32; + break; + case PKEY_KEYTYPE_ECC_P384: + keylen = 48; + break; + case PKEY_KEYTYPE_ECC_P521: + keylen = 80; + break; + case PKEY_KEYTYPE_ECC_ED25519: + keylen = 32; + break; + case PKEY_KEYTYPE_ECC_ED448: + keylen = 64; + break; + default: + DEBUG_ERR("%s unknown/unsupported keytype %u\n", + __func__, t->keytype); + return -EINVAL; + } + + if (t->len != keylen) { + DEBUG_ERR("%s non clear key ecc token: invalid key len %u\n", + __func__, t->len); + return -EINVAL; + } + + /* only one path possible: via PCKMO instruction */ + rc = pkey_clr2protkey(t->keytype, t->clearkey, + protkey, protkeylen, protkeytype); + if (rc) { + DEBUG_ERR("%s unable to build protected key from clear", + __func__); + } + + return rc; +} + /* * Transform a non-CCA key token into a protected key */ @@ -411,7 +561,6 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, u8 *protkey, u32 *protkeylen, u32 *protkeytype) { struct keytoken_header *hdr = (struct keytoken_header *)key; - u8 *tmpbuf = NULL; int rc = -EINVAL; switch (hdr->version) { @@ -430,54 +579,31 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, break; } case TOKVER_CLEAR_KEY: { - struct clearaeskeytoken *t; - struct pkey_clrkey ckey; - union u_tmpbuf { - u8 skey[SECKEYBLOBSIZE]; - u8 ep11key[MAXEP11AESKEYBLOBSIZE]; - }; - size_t tmpbuflen = sizeof(union u_tmpbuf); - - if (keylen < sizeof(struct clearaeskeytoken)) - goto out; - t = (struct clearaeskeytoken *)key; - if (keylen != sizeof(*t) + t->len) - goto out; - if ((t->keytype == PKEY_KEYTYPE_AES_128 && t->len == 16) || - (t->keytype == PKEY_KEYTYPE_AES_192 && t->len == 24) || - (t->keytype == PKEY_KEYTYPE_AES_256 && t->len == 32)) - memcpy(ckey.clrkey, t->clearkey, t->len); - else - goto out; - /* alloc temp key buffer space */ - tmpbuf = kmalloc(tmpbuflen, GFP_ATOMIC); - if (!tmpbuf) { - rc = -ENOMEM; + struct clearkeytoken *t = (struct clearkeytoken *)key; + + if (keylen < sizeof(struct clearkeytoken) || + keylen != sizeof(*t) + t->len) goto out; - } - /* try direct way with the PCKMO instruction */ - rc = pkey_clr2protkey(t->keytype, ckey.clrkey, - protkey, protkeylen, protkeytype); - if (rc == 0) + switch (t->keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + rc = nonccatokaes2pkey(t, protkey, + protkeylen, protkeytype); break; - /* PCKMO failed, so try the CCA secure key way */ - zcrypt_wait_api_operational(); - rc = cca_clr2seckey(0xFFFF, 0xFFFF, t->keytype, - ckey.clrkey, tmpbuf); - if (rc == 0) - rc = pkey_skey2pkey(tmpbuf, - protkey, protkeylen, protkeytype); - if (rc == 0) + case PKEY_KEYTYPE_ECC_P256: + case PKEY_KEYTYPE_ECC_P384: + case PKEY_KEYTYPE_ECC_P521: + case PKEY_KEYTYPE_ECC_ED25519: + case PKEY_KEYTYPE_ECC_ED448: + rc = nonccatokecc2pkey(t, protkey, + protkeylen, protkeytype); break; - /* if the CCA way also failed, let's try via EP11 */ - rc = pkey_clr2ep11key(ckey.clrkey, t->len, - tmpbuf, &tmpbuflen); - if (rc == 0) - rc = pkey_ep11key2pkey(tmpbuf, - protkey, protkeylen, protkeytype); - /* now we should really have an protected key */ - DEBUG_ERR("%s unable to build protected key from clear", - __func__); + default: + DEBUG_ERR("%s unknown/unsupported non cca clear key type %u\n", + __func__, t->keytype); + return -EINVAL; + } break; } case TOKVER_EP11_AES: { @@ -500,11 +626,9 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, default: DEBUG_ERR("%s unknown/unsupported non-CCA token version %d\n", __func__, hdr->version); - rc = -EINVAL; } out: - kfree(tmpbuf); return rc; } @@ -1149,7 +1273,7 @@ static int pkey_keyblob2pkey3(const struct pkey_apqn *apqns, size_t nr_apqns, static void *_copy_key_from_user(void __user *ukey, size_t keylen) { - if (!ukey || keylen < MINKEYBLOBSIZE || keylen > KEYBLOBBUFSIZE) + if (!ukey || keylen < MINKEYBLOBBUFSIZE || keylen > KEYBLOBBUFSIZE) return ERR_PTR(-EINVAL); return memdup_user(ukey, keylen); -- cgit v1.2.3 From 6afc770048edc90405c444163de70b1cdfbb8b57 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Tue, 30 May 2023 18:35:36 -0400 Subject: s390/vfio-ap: realize the VFIO_DEVICE_GET_IRQ_INFO ioctl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Realize the VFIO_DEVICE_GET_IRQ_INFO ioctl to retrieve the information for the VFIO device request IRQ. Signed-off-by: Tony Krowiak Acked-by: Alex Williamson Reviewed-by: Cédric Le Goater Reviewed-by: Matthew Rosato Tested-by: Matthew Rosato Link: https://lore.kernel.org/r/20230530223538.279198-2-akrowiak@linux.ibm.com Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 30 +++++++++++++++++++++++++++++- include/uapi/linux/vfio.h | 9 +++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index cfbcb864ab63..35cd90eee937 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -1750,7 +1750,32 @@ static int vfio_ap_mdev_get_device_info(unsigned long arg) info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET; info.num_regions = 0; - info.num_irqs = 0; + info.num_irqs = VFIO_AP_NUM_IRQS; + + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; +} + +static ssize_t vfio_ap_get_irq_info(unsigned long arg) +{ + unsigned long minsz; + struct vfio_irq_info info; + + minsz = offsetofend(struct vfio_irq_info, count); + + if (copy_from_user(&info, (void __user *)arg, minsz)) + return -EFAULT; + + if (info.argsz < minsz || info.index >= VFIO_AP_NUM_IRQS) + return -EINVAL; + + switch (info.index) { + case VFIO_AP_REQ_IRQ_INDEX: + info.count = 1; + info.flags = VFIO_IRQ_INFO_EVENTFD; + break; + default: + return -EINVAL; + } return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; } @@ -1770,6 +1795,9 @@ static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, case VFIO_DEVICE_RESET: ret = vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); break; + case VFIO_DEVICE_GET_IRQ_INFO: + ret = vfio_ap_get_irq_info(arg); + break; default: ret = -EOPNOTSUPP; break; diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 0552e8dcf0cb..b71276bd7f91 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -646,6 +646,15 @@ enum { VFIO_CCW_NUM_IRQS }; +/* + * The vfio-ap bus driver makes use of the following IRQ index mapping. + * Unimplemented IRQ types return a count of zero. + */ +enum { + VFIO_AP_REQ_IRQ_INDEX, + VFIO_AP_NUM_IRQS +}; + /** * VFIO_DEVICE_GET_PCI_HOT_RESET_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 12, * struct vfio_pci_hot_reset_info) -- cgit v1.2.3 From bf48961f6f48e3b7eb80c3e179207e9f4e4cd660 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Tue, 30 May 2023 18:35:37 -0400 Subject: s390/vfio-ap: realize the VFIO_DEVICE_SET_IRQS ioctl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Realize the VFIO_DEVICE_SET_IRQS ioctl to set an eventfd file descriptor to be used by the vfio_ap device driver to signal a device request to userspace. Signed-off-by: Tony Krowiak Reviewed-by: Cédric Le Goater Reviewed-by: Matthew Rosato Tested-by: Matthew Rosato Link: https://lore.kernel.org/r/20230530223538.279198-3-akrowiak@linux.ibm.com Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 83 +++++++++++++++++++++++++++++++++++ drivers/s390/crypto/vfio_ap_private.h | 3 ++ 2 files changed, 86 insertions(+) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 35cd90eee937..44f159136891 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -716,6 +716,7 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev) ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev); if (ret) goto err_put_vdev; + matrix_mdev->req_trigger = NULL; dev_set_drvdata(&mdev->dev, matrix_mdev); mutex_lock(&matrix_dev->mdevs_lock); list_add(&matrix_mdev->node, &matrix_dev->mdev_list); @@ -1780,6 +1781,85 @@ static ssize_t vfio_ap_get_irq_info(unsigned long arg) return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; } +static int vfio_ap_irq_set_init(struct vfio_irq_set *irq_set, unsigned long arg) +{ + int ret; + size_t data_size; + unsigned long minsz; + + minsz = offsetofend(struct vfio_irq_set, count); + + if (copy_from_user(irq_set, (void __user *)arg, minsz)) + return -EFAULT; + + ret = vfio_set_irqs_validate_and_prepare(irq_set, 1, VFIO_AP_NUM_IRQS, + &data_size); + if (ret) + return ret; + + if (!(irq_set->flags & VFIO_IRQ_SET_ACTION_TRIGGER)) + return -EINVAL; + + return 0; +} + +static int vfio_ap_set_request_irq(struct ap_matrix_mdev *matrix_mdev, + unsigned long arg) +{ + s32 fd; + void __user *data; + unsigned long minsz; + struct eventfd_ctx *req_trigger; + + minsz = offsetofend(struct vfio_irq_set, count); + data = (void __user *)(arg + minsz); + + if (get_user(fd, (s32 __user *)data)) + return -EFAULT; + + if (fd == -1) { + if (matrix_mdev->req_trigger) + eventfd_ctx_put(matrix_mdev->req_trigger); + matrix_mdev->req_trigger = NULL; + } else if (fd >= 0) { + req_trigger = eventfd_ctx_fdget(fd); + if (IS_ERR(req_trigger)) + return PTR_ERR(req_trigger); + + if (matrix_mdev->req_trigger) + eventfd_ctx_put(matrix_mdev->req_trigger); + + matrix_mdev->req_trigger = req_trigger; + } else { + return -EINVAL; + } + + return 0; +} + +static int vfio_ap_set_irqs(struct ap_matrix_mdev *matrix_mdev, + unsigned long arg) +{ + int ret; + struct vfio_irq_set irq_set; + + ret = vfio_ap_irq_set_init(&irq_set, arg); + if (ret) + return ret; + + switch (irq_set.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { + case VFIO_IRQ_SET_DATA_EVENTFD: + switch (irq_set.index) { + case VFIO_AP_REQ_IRQ_INDEX: + return vfio_ap_set_request_irq(matrix_mdev, arg); + default: + return -EINVAL; + } + default: + return -EINVAL; + } +} + static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, unsigned int cmd, unsigned long arg) { @@ -1798,6 +1878,9 @@ static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, case VFIO_DEVICE_GET_IRQ_INFO: ret = vfio_ap_get_irq_info(arg); break; + case VFIO_DEVICE_SET_IRQS: + ret = vfio_ap_set_irqs(matrix_mdev, arg); + break; default: ret = -EOPNOTSUPP; break; diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index 976a65f32e7d..4642bbdbd1b2 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -103,6 +104,7 @@ struct ap_queue_table { * PQAP(AQIC) instruction. * @mdev: the mediated device * @qtable: table of queues (struct vfio_ap_queue) assigned to the mdev + * @req_trigger eventfd ctx for signaling userspace to return a device * @apm_add: bitmap of APIDs added to the host's AP configuration * @aqm_add: bitmap of APQIs added to the host's AP configuration * @adm_add: bitmap of control domain numbers added to the host's AP @@ -117,6 +119,7 @@ struct ap_matrix_mdev { crypto_hook pqap_hook; struct mdev_device *mdev; struct ap_queue_table qtable; + struct eventfd_ctx *req_trigger; DECLARE_BITMAP(apm_add, AP_DEVICES); DECLARE_BITMAP(aqm_add, AP_DOMAINS); DECLARE_BITMAP(adm_add, AP_DOMAINS); -- cgit v1.2.3 From 2e3d8d71e285fcf39eb30dbb17a58baa90649867 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Tue, 30 May 2023 18:35:38 -0400 Subject: s390/vfio-ap: wire in the vfio_device_ops request callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mdev device is being removed, so pass the request to userspace to ask for a graceful cleanup. This should free up the thread that would otherwise loop waiting for the device to be fully released. Signed-off-by: Tony Krowiak Reviewed-by: Cédric Le Goater Reviewed-by: Matthew Rosato Tested-by: Matthew Rosato Acked-by: Alex Williamson Link: https://lore.kernel.org/r/20230530223538.279198-4-akrowiak@linux.ibm.com Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 44f159136891..a8f58e133e6e 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -1736,6 +1736,26 @@ static void vfio_ap_mdev_close_device(struct vfio_device *vdev) vfio_ap_mdev_unset_kvm(matrix_mdev); } +static void vfio_ap_mdev_request(struct vfio_device *vdev, unsigned int count) +{ + struct device *dev = vdev->dev; + struct ap_matrix_mdev *matrix_mdev; + + matrix_mdev = container_of(vdev, struct ap_matrix_mdev, vdev); + + if (matrix_mdev->req_trigger) { + if (!(count % 10)) + dev_notice_ratelimited(dev, + "Relaying device request to user (#%u)\n", + count); + + eventfd_signal(matrix_mdev->req_trigger, 1); + } else if (count == 0) { + dev_notice(dev, + "No device request registered, blocked until released by user\n"); + } +} + static int vfio_ap_mdev_get_device_info(unsigned long arg) { unsigned long minsz; @@ -1955,6 +1975,7 @@ static const struct vfio_device_ops vfio_ap_matrix_dev_ops = { .bind_iommufd = vfio_iommufd_emulated_bind, .unbind_iommufd = vfio_iommufd_emulated_unbind, .attach_ioas = vfio_iommufd_emulated_attach_ioas, + .request = vfio_ap_mdev_request }; static struct mdev_driver vfio_ap_matrix_driver = { -- cgit v1.2.3 From 11458e2b3ffa0e3798f392695d7aec210ac14efd Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Mon, 5 Jun 2023 09:55:27 +0200 Subject: s390/module: fix rela calculation for R_390_GOTENT During module load, module layout allocation occurs by initially allowing the architecture to frob the sections. This is performed via module_frob_arch_sections(). However, the size of each module memory types like text,data,rodata etc are updated correctly only after layout_sections(). After calculation of required module memory sizes for each types, move_module() is responsible for allocating the module memory for each type from modules vaddr range. Considering the sequence above, module_frob_arch_sections() updates the module mod_arch_specific got_offset before module memory text type size is fully updated in layout_sections(). Hence mod_arch_specific got_offset points to currently zero. As per s390 ABI, R_390_GOTENT : (G + O + A - P) >> 1 where G=me->mem[MOD_TEXT].base+me->arch.got_offset O=info->got_offset A=rela->r_addend P=loc fix R_390_GOTENT calculation in apply_rela(). Note: currently this doesn't break anything because me->arch.got_offset is zero. However, reordering of functions in the future could break it. Signed-off-by: Sumanth Korikkar Acked-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/kernel/module.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index f1b35dcdf3eb..42215f9404af 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -352,7 +352,8 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, rc = apply_rela_bits(loc, val, 0, 64, 0, write); else if (r_type == R_390_GOTENT || r_type == R_390_GOTPLTENT) { - val += (Elf_Addr) me->mem[MOD_TEXT].base - loc; + val += (Elf_Addr)me->mem[MOD_TEXT].base + + me->arch.got_offset - loc; rc = apply_rela_bits(loc, val, 1, 32, 1, write); } break; -- cgit v1.2.3 From d0d3e218d50bc93282df7bded9193e6fc9ccda48 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Wed, 14 Jun 2023 09:37:42 +0200 Subject: s390/cpum_cf: open access to hwctr device for CAP_PERFMON privileged process The device /dev/hwctr was introduced to access complete CPU Measurement facility counter sets via an ioctl system call. The access the to device is limited to privileged processes running as root or superuser. The capability CAP_SYS_ADMIN is required. The device permissions are read/write for the device owner root. There is no need for this restriction. Make the device access permission read/write for all and reduce the capabilities to CAP_PERFMON. Any user space program with the CAP_PERFMON capability assigned to it can now read and display the CPU Measurement facility counter sets. For more details on perf tool usage and security, see linux documentation in Documentation/admin-guide/perf-security.rst. Signed-off-by: Thomas Richter Acked-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/kernel/perf_cpum_cf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index cf1b6e8a708d..6be0387d77e4 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -1213,7 +1213,7 @@ static int cfset_release(struct inode *inode, struct file *file) static int cfset_open(struct inode *inode, struct file *file) { - if (!capable(CAP_SYS_ADMIN)) + if (!perfmon_capable()) return -EPERM; mutex_lock(&cfset_ctrset_mutex); if (atomic_inc_return(&cfset_opencnt) == 1) @@ -1502,6 +1502,7 @@ static struct miscdevice cfset_dev = { .name = S390_HWCTR_DEVICE, .minor = MISC_DYNAMIC_MINOR, .fops = &cfset_fops, + .mode = 0666, }; /* Hotplug add of a CPU. Scan through all active processes and add -- cgit v1.2.3 From 9b9cf3c77e7e090b30657b3d2c288deb58dfb4f2 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 15 Jun 2023 13:31:58 +0200 Subject: s390/cpum_cf: rework PER_CPU_DEFINE of struct cpu_cf_events Struct cpu_cf_events is a large data structure and is statically defined for each possible CPU. Rework this and replace it by dynamically allocated data structures created when a perf_event_open() system call is invoked or an access via character device /dev/hwctr takes place. It is replaced by an array of pointers to all possible CPUs and reference counting. The array of pointers is allocated when the first event is created. For each online CPU an event is installed on, a struct cpu_cf_events is allocated and a pointer to struct cpu_cf_events is stored in the array: CPU 0 1 2 3 ... N +---+---+---+---+---+---+ cpu_cf_root::cpucf--> | * | | | |...| | +-|-+---+---+---+---+---+ | | \|/ +-------------+ |cpu_cf_events| | | +-------------+ With this approach the large data structure is only allocated when an event is actually installed and used. Also implement proper reference counting for allocation and removal. During interrupt processing make sure the pointer to cpu_cf_events is valid. The interrupt handler is shared and might be called when no event is active. This requires checking for a valid pointer to struct cpu_cf_events. When the pointer to the per-cpu cpu_cf_events is NULL, simply return. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Acked-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/kernel/perf_cpum_cf.c | 449 +++++++++++++++++++++++++++++----------- 1 file changed, 332 insertions(+), 117 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 6be0387d77e4..90679143534b 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -76,6 +76,7 @@ static inline int ctr_stcctm(enum cpumf_ctr_set set, u64 range, u64 *dest) } struct cpu_cf_events { + refcount_t refcnt; /* Reference count */ atomic_t ctr_set[CPUMF_CTR_SET_MAX]; u64 state; /* For perf_event_open SVC */ u64 dev_state; /* For /dev/hwctr */ @@ -88,9 +89,6 @@ struct cpu_cf_events { unsigned int sets; /* # Counter set saved in memory */ }; -/* Per-CPU event structure for the counter facility */ -static DEFINE_PER_CPU(struct cpu_cf_events, cpu_cf_events); - static unsigned int cfdiag_cpu_speed; /* CPU speed for CF_DIAG trailer */ static debug_info_t *cf_dbg; @@ -103,6 +101,221 @@ static debug_info_t *cf_dbg; */ static struct cpumf_ctr_info cpumf_ctr_info; +struct cpu_cf_ptr { + struct cpu_cf_events *cpucf; +}; + +static struct cpu_cf_root { /* Anchor to per CPU data */ + refcount_t refcnt; /* Overall active events */ + struct cpu_cf_ptr __percpu *cfptr; +} cpu_cf_root; + +/* + * Serialize event initialization and event removal. Both are called from + * user space in task context with perf_event_open() and close() + * system calls. + * + * This mutex serializes functions cpum_cf_alloc_cpu() called at event + * initialization via cpumf_pmu_event_init() and function cpum_cf_free_cpu() + * called at event removal via call back function hw_perf_event_destroy() + * when the event is deleted. They are serialized to enforce correct + * bookkeeping of pointer and reference counts anchored by + * struct cpu_cf_root and the access to cpu_cf_root::refcnt and the + * per CPU pointers stored in cpu_cf_root::cfptr. + */ +static DEFINE_MUTEX(pmc_reserve_mutex); + +/* + * Get pointer to per-cpu structure. + * + * Function get_cpu_cfhw() is called from + * - cfset_copy_all(): This function is protected by cpus_read_lock(), so + * CPU hot plug remove can not happen. Event removal requires a close() + * first. + * + * Function this_cpu_cfhw() is called from perf common code functions: + * - pmu_{en|dis}able(), pmu_{add|del}()and pmu_{start|stop}(): + * All functions execute with interrupts disabled on that particular CPU. + * - cfset_ioctl_{on|off}, cfset_cpu_read(): see comment cfset_copy_all(). + * + * Therefore it is safe to access the CPU specific pointer to the event. + */ +static struct cpu_cf_events *get_cpu_cfhw(int cpu) +{ + struct cpu_cf_ptr __percpu *p = cpu_cf_root.cfptr; + + if (p) { + struct cpu_cf_ptr *q = per_cpu_ptr(p, cpu); + + return q->cpucf; + } + return NULL; +} + +static struct cpu_cf_events *this_cpu_cfhw(void) +{ + return get_cpu_cfhw(smp_processor_id()); +} + +/* Disable counter sets on dedicated CPU */ +static void cpum_cf_reset_cpu(void *flags) +{ + lcctl(0); +} + +/* Free per CPU data when the last event is removed. */ +static void cpum_cf_free_root(void) +{ + if (!refcount_dec_and_test(&cpu_cf_root.refcnt)) + return; + free_percpu(cpu_cf_root.cfptr); + cpu_cf_root.cfptr = NULL; + irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); + on_each_cpu(cpum_cf_reset_cpu, NULL, 1); + debug_sprintf_event(cf_dbg, 4, "%s2 root.refcnt %u cfptr %px\n", + __func__, refcount_read(&cpu_cf_root.refcnt), + cpu_cf_root.cfptr); +} + +/* + * On initialization of first event also allocate per CPU data dynamically. + * Start with an array of pointers, the array size is the maximum number of + * CPUs possible, which might be larger than the number of CPUs currently + * online. + */ +static int cpum_cf_alloc_root(void) +{ + int rc = 0; + + if (refcount_inc_not_zero(&cpu_cf_root.refcnt)) + return rc; + + /* The memory is already zeroed. */ + cpu_cf_root.cfptr = alloc_percpu(struct cpu_cf_ptr); + if (cpu_cf_root.cfptr) { + refcount_set(&cpu_cf_root.refcnt, 1); + on_each_cpu(cpum_cf_reset_cpu, NULL, 1); + irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); + } else { + rc = -ENOMEM; + } + + return rc; +} + +/* Free CPU counter data structure for a PMU */ +static void cpum_cf_free_cpu(int cpu) +{ + struct cpu_cf_events *cpuhw; + struct cpu_cf_ptr *p; + + mutex_lock(&pmc_reserve_mutex); + /* + * When invoked via CPU hotplug handler, there might be no events + * installed or that particular CPU might not have an + * event installed. This anchor pointer can be NULL! + */ + if (!cpu_cf_root.cfptr) + goto out; + p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); + cpuhw = p->cpucf; + /* + * Might be zero when called from CPU hotplug handler and no event + * installed on that CPU, but on different CPUs. + */ + if (!cpuhw) + goto out; + + if (refcount_dec_and_test(&cpuhw->refcnt)) { + kfree(cpuhw); + p->cpucf = NULL; + } + cpum_cf_free_root(); +out: + mutex_unlock(&pmc_reserve_mutex); +} + +/* Allocate CPU counter data structure for a PMU. Called under mutex lock. */ +static int cpum_cf_alloc_cpu(int cpu) +{ + struct cpu_cf_events *cpuhw; + struct cpu_cf_ptr *p; + int rc; + + mutex_lock(&pmc_reserve_mutex); + rc = cpum_cf_alloc_root(); + if (rc) + goto unlock; + p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); + cpuhw = p->cpucf; + + if (!cpuhw) { + cpuhw = kzalloc(sizeof(*cpuhw), GFP_KERNEL); + if (cpuhw) { + p->cpucf = cpuhw; + refcount_set(&cpuhw->refcnt, 1); + } else { + rc = -ENOMEM; + } + } else { + refcount_inc(&cpuhw->refcnt); + } + if (rc) { + /* + * Error in allocation of event, decrement anchor. Since + * cpu_cf_event in not created, its destroy() function is not + * invoked. Adjust the reference counter for the anchor. + */ + cpum_cf_free_root(); + } +unlock: + mutex_unlock(&pmc_reserve_mutex); + return rc; +} + +/* + * Create/delete per CPU data structures for /dev/hwctr interface and events + * created by perf_event_open(). + * If cpu is -1, track task on all available CPUs. This requires + * allocation of hardware data structures for all CPUs. This setup handles + * perf_event_open() with task context and /dev/hwctr interface. + * If cpu is non-zero install event on this CPU only. This setup handles + * perf_event_open() with CPU context. + */ +static int cpum_cf_alloc(int cpu) +{ + cpumask_var_t mask; + int rc; + + if (cpu == -1) { + if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) + return -ENOMEM; + for_each_online_cpu(cpu) { + rc = cpum_cf_alloc_cpu(cpu); + if (rc) { + for_each_cpu(cpu, mask) + cpum_cf_free_cpu(cpu); + break; + } + cpumask_set_cpu(cpu, mask); + } + free_cpumask_var(mask); + } else { + rc = cpum_cf_alloc_cpu(cpu); + } + return rc; +} + +static void cpum_cf_free(int cpu) +{ + if (cpu == -1) { + for_each_online_cpu(cpu) + cpum_cf_free_cpu(cpu); + } else { + cpum_cf_free_cpu(cpu); + } +} + #define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ /* interval in seconds */ @@ -451,10 +664,10 @@ static int validate_ctr_version(const u64 config, enum cpumf_ctr_set set) */ static void cpumf_pmu_enable(struct pmu *pmu) { - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cpu_cf_events *cpuhw = this_cpu_cfhw(); int err; - if (cpuhw->flags & PMU_F_ENABLED) + if (!cpuhw || (cpuhw->flags & PMU_F_ENABLED)) return; err = lcctl(cpuhw->state | cpuhw->dev_state); @@ -471,11 +684,11 @@ static void cpumf_pmu_enable(struct pmu *pmu) */ static void cpumf_pmu_disable(struct pmu *pmu) { - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); - int err; + struct cpu_cf_events *cpuhw = this_cpu_cfhw(); u64 inactive; + int err; - if (!(cpuhw->flags & PMU_F_ENABLED)) + if (!cpuhw || !(cpuhw->flags & PMU_F_ENABLED)) return; inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1); @@ -487,58 +700,10 @@ static void cpumf_pmu_disable(struct pmu *pmu) cpuhw->flags &= ~PMU_F_ENABLED; } -#define PMC_INIT 0UL -#define PMC_RELEASE 1UL - -static void cpum_cf_setup_cpu(void *flags) -{ - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); - - switch ((unsigned long)flags) { - case PMC_INIT: - cpuhw->flags |= PMU_F_RESERVED; - break; - - case PMC_RELEASE: - cpuhw->flags &= ~PMU_F_RESERVED; - break; - } - - /* Disable CPU counter sets */ - lcctl(0); - debug_sprintf_event(cf_dbg, 5, "%s flags %#x flags %#x state %#llx\n", - __func__, *(int *)flags, cpuhw->flags, - cpuhw->state); -} - -/* Initialize the CPU-measurement counter facility */ -static int __kernel_cpumcf_begin(void) -{ - on_each_cpu(cpum_cf_setup_cpu, (void *)PMC_INIT, 1); - irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); - - return 0; -} - -/* Release the CPU-measurement counter facility */ -static void __kernel_cpumcf_end(void) -{ - on_each_cpu(cpum_cf_setup_cpu, (void *)PMC_RELEASE, 1); - irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); -} - -/* Number of perf events counting hardware events */ -static atomic_t num_events = ATOMIC_INIT(0); -/* Used to avoid races in calling reserve/release_cpumf_hardware */ -static DEFINE_MUTEX(pmc_reserve_mutex); - /* Release the PMU if event is the last perf event */ static void hw_perf_event_destroy(struct perf_event *event) { - mutex_lock(&pmc_reserve_mutex); - if (atomic_dec_return(&num_events) == 0) - __kernel_cpumcf_end(); - mutex_unlock(&pmc_reserve_mutex); + cpum_cf_free(event->cpu); } /* CPUMF <-> perf event mappings for kernel+userspace (basic set) */ @@ -562,14 +727,6 @@ static const int cpumf_generic_events_user[] = { [PERF_COUNT_HW_BUS_CYCLES] = -1, }; -static void cpumf_hw_inuse(void) -{ - mutex_lock(&pmc_reserve_mutex); - if (atomic_inc_return(&num_events) == 1) - __kernel_cpumcf_begin(); - mutex_unlock(&pmc_reserve_mutex); -} - static int is_userspace_event(u64 ev) { return cpumf_generic_events_user[PERF_COUNT_HW_CPU_CYCLES] == ev || @@ -653,7 +810,8 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type) } /* Initialize for using the CPU-measurement counter facility */ - cpumf_hw_inuse(); + if (cpum_cf_alloc(event->cpu)) + return -ENOMEM; event->destroy = hw_perf_event_destroy; /* @@ -756,7 +914,7 @@ static void cpumf_pmu_read(struct perf_event *event) static void cpumf_pmu_start(struct perf_event *event, int flags) { - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cpu_cf_events *cpuhw = this_cpu_cfhw(); struct hw_perf_event *hwc = &event->hw; int i; @@ -830,7 +988,7 @@ static int cfdiag_push_sample(struct perf_event *event, static void cpumf_pmu_stop(struct perf_event *event, int flags) { - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cpu_cf_events *cpuhw = this_cpu_cfhw(); struct hw_perf_event *hwc = &event->hw; int i; @@ -857,8 +1015,7 @@ static void cpumf_pmu_stop(struct perf_event *event, int flags) false); if (cfdiag_diffctr(cpuhw, event->hw.config_base)) cfdiag_push_sample(event, cpuhw); - } else if (cpuhw->flags & PMU_F_RESERVED) { - /* Only update when PMU not hotplugged off */ + } else { hw_perf_event_update(event); } hwc->state |= PERF_HES_UPTODATE; @@ -867,7 +1024,7 @@ static void cpumf_pmu_stop(struct perf_event *event, int flags) static int cpumf_pmu_add(struct perf_event *event, int flags) { - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cpu_cf_events *cpuhw = this_cpu_cfhw(); ctr_set_enable(&cpuhw->state, event->hw.config_base); event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; @@ -880,7 +1037,7 @@ static int cpumf_pmu_add(struct perf_event *event, int flags) static void cpumf_pmu_del(struct perf_event *event, int flags) { - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cpu_cf_events *cpuhw = this_cpu_cfhw(); int i; cpumf_pmu_stop(event, PERF_EF_UPDATE); @@ -912,29 +1069,83 @@ static struct pmu cpumf_pmu = { .read = cpumf_pmu_read, }; -static int cpum_cf_setup(unsigned int cpu, unsigned long flags) -{ - local_irq_disable(); - cpum_cf_setup_cpu((void *)flags); - local_irq_enable(); - return 0; -} +static struct cfset_session { /* CPUs and counter set bit mask */ + struct list_head head; /* Head of list of active processes */ +} cfset_session = { + .head = LIST_HEAD_INIT(cfset_session.head) +}; + +static refcount_t cfset_opencnt = REFCOUNT_INIT(0); /* Access count */ +/* + * Synchronize access to device /dev/hwc. This mutex protects against + * concurrent access to functions cfset_open() and cfset_release(). + * Same for CPU hotplug add and remove events triggering + * cpum_cf_online_cpu() and cpum_cf_offline_cpu(). + * It also serializes concurrent device ioctl access from multiple + * processes accessing /dev/hwc. + * + * The mutex protects concurrent access to the /dev/hwctr session management + * struct cfset_session and reference counting variable cfset_opencnt. + */ +static DEFINE_MUTEX(cfset_ctrset_mutex); +/* + * CPU hotplug handles only /dev/hwctr device. + * For perf_event_open() the CPU hotplug handling is done on kernel common + * code: + * - CPU add: Nothing is done since a file descriptor can not be created + * and returned to the user. + * - CPU delete: Handled by common code via pmu_disable(), pmu_stop() and + * pmu_delete(). The event itself is removed when the file descriptor is + * closed. + */ static int cfset_online_cpu(unsigned int cpu); + static int cpum_cf_online_cpu(unsigned int cpu) { - debug_sprintf_event(cf_dbg, 4, "%s cpu %d in_irq %ld\n", __func__, - cpu, in_interrupt()); - cpum_cf_setup(cpu, PMC_INIT); - return cfset_online_cpu(cpu); + int rc = 0; + + debug_sprintf_event(cf_dbg, 4, "%s cpu %d root.refcnt %d " + "opencnt %d\n", __func__, cpu, + refcount_read(&cpu_cf_root.refcnt), + refcount_read(&cfset_opencnt)); + /* + * Ignore notification for perf_event_open(). + * Handle only /dev/hwctr device sessions. + */ + mutex_lock(&cfset_ctrset_mutex); + if (refcount_read(&cfset_opencnt)) { + rc = cpum_cf_alloc_cpu(cpu); + if (!rc) + cfset_online_cpu(cpu); + } + mutex_unlock(&cfset_ctrset_mutex); + return rc; } static int cfset_offline_cpu(unsigned int cpu); + static int cpum_cf_offline_cpu(unsigned int cpu) { - debug_sprintf_event(cf_dbg, 4, "%s cpu %d\n", __func__, cpu); - cfset_offline_cpu(cpu); - return cpum_cf_setup(cpu, PMC_RELEASE); + debug_sprintf_event(cf_dbg, 4, "%s cpu %d root.refcnt %d opencnt %d\n", + __func__, cpu, refcount_read(&cpu_cf_root.refcnt), + refcount_read(&cfset_opencnt)); + /* + * During task exit processing of grouped perf events triggered by CPU + * hotplug processing, pmu_disable() is called as part of perf context + * removal process. Therefore do not trigger event removal now for + * perf_event_open() created events. Perf common code triggers event + * destruction when the event file descriptor is closed. + * + * Handle only /dev/hwctr device sessions. + */ + mutex_lock(&cfset_ctrset_mutex); + if (refcount_read(&cfset_opencnt)) { + cfset_offline_cpu(cpu); + cpum_cf_free_cpu(cpu); + } + mutex_unlock(&cfset_ctrset_mutex); + return 0; } /* Return true if store counter set multiple instruction is available */ @@ -953,13 +1164,13 @@ static void cpumf_measurement_alert(struct ext_code ext_code, return; inc_irq_stat(IRQEXT_CMC); - cpuhw = this_cpu_ptr(&cpu_cf_events); /* * Measurement alerts are shared and might happen when the PMU * is not reserved. Ignore these alerts in this case. */ - if (!(cpuhw->flags & PMU_F_RESERVED)) + cpuhw = this_cpu_cfhw(); + if (!cpuhw) return; /* counter authorization change alert */ @@ -1039,19 +1250,11 @@ out1: * counter set via normal file operations. */ -static atomic_t cfset_opencnt = ATOMIC_INIT(0); /* Access count */ -static DEFINE_MUTEX(cfset_ctrset_mutex);/* Synchronize access to hardware */ struct cfset_call_on_cpu_parm { /* Parm struct for smp_call_on_cpu */ unsigned int sets; /* Counter set bit mask */ atomic_t cpus_ack; /* # CPUs successfully executed func */ }; -static struct cfset_session { /* CPUs and counter set bit mask */ - struct list_head head; /* Head of list of active processes */ -} cfset_session = { - .head = LIST_HEAD_INIT(cfset_session.head) -}; - struct cfset_request { /* CPUs and counter set bit mask */ unsigned long ctrset; /* Bit mask of counter set to read */ cpumask_t mask; /* CPU mask to read from */ @@ -1113,11 +1316,11 @@ static void cfset_session_add(struct cfset_request *p) /* Stop all counter sets via ioctl interface */ static void cfset_ioctl_off(void *parm) { - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cpu_cf_events *cpuhw = this_cpu_cfhw(); struct cfset_call_on_cpu_parm *p = parm; int rc; - /* Check if any counter set used by /dev/hwc */ + /* Check if any counter set used by /dev/hwctr */ for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc) if ((p->sets & cpumf_ctr_ctl[rc])) { if (!atomic_dec_return(&cpuhw->ctr_set[rc])) { @@ -1141,7 +1344,7 @@ static void cfset_ioctl_off(void *parm) /* Start counter sets on particular CPU */ static void cfset_ioctl_on(void *parm) { - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cpu_cf_events *cpuhw = this_cpu_cfhw(); struct cfset_call_on_cpu_parm *p = parm; int rc; @@ -1163,7 +1366,7 @@ static void cfset_ioctl_on(void *parm) static void cfset_release_cpu(void *p) { - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cpu_cf_events *cpuhw = this_cpu_cfhw(); int rc; debug_sprintf_event(cf_dbg, 4, "%s state %#llx dev_state %#llx\n", @@ -1203,27 +1406,41 @@ static int cfset_release(struct inode *inode, struct file *file) kfree(file->private_data); file->private_data = NULL; } - if (!atomic_dec_return(&cfset_opencnt)) + if (refcount_dec_and_test(&cfset_opencnt)) { /* Last close */ on_each_cpu(cfset_release_cpu, NULL, 1); + cpum_cf_free(-1); + } mutex_unlock(&cfset_ctrset_mutex); - - hw_perf_event_destroy(NULL); return 0; } +/* + * Open via /dev/hwctr device. Allocate all per CPU resources on the first + * open of the device. The last close releases all per CPU resources. + * Parallel perf_event_open system calls also use per CPU resources. + * These invocations are handled via reference counting on the per CPU data + * structures. + */ static int cfset_open(struct inode *inode, struct file *file) { + int rc = 0; + if (!perfmon_capable()) return -EPERM; + file->private_data = NULL; + mutex_lock(&cfset_ctrset_mutex); - if (atomic_inc_return(&cfset_opencnt) == 1) - cfset_session_init(); + if (!refcount_inc_not_zero(&cfset_opencnt)) { /* First open */ + rc = cpum_cf_alloc(-1); + if (!rc) { + cfset_session_init(); + refcount_set(&cfset_opencnt, 1); + } + } mutex_unlock(&cfset_ctrset_mutex); - cpumf_hw_inuse(); - file->private_data = NULL; /* nonseekable_open() never fails */ - return nonseekable_open(inode, file); + return rc ?: nonseekable_open(inode, file); } static int cfset_all_start(struct cfset_request *req) @@ -1280,7 +1497,7 @@ static int cfset_all_copy(unsigned long arg, cpumask_t *mask) ctrset_read = (struct s390_ctrset_read __user *)arg; uptr = ctrset_read->data; for_each_cpu(cpu, mask) { - struct cpu_cf_events *cpuhw = per_cpu_ptr(&cpu_cf_events, cpu); + struct cpu_cf_events *cpuhw = get_cpu_cfhw(cpu); struct s390_ctrset_cpudata __user *ctrset_cpudata; ctrset_cpudata = uptr; @@ -1324,7 +1541,7 @@ static size_t cfset_cpuset_read(struct s390_ctrset_setdata *p, int ctrset, /* Read all counter sets. */ static void cfset_cpu_read(void *parm) { - struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cpu_cf_events *cpuhw = this_cpu_cfhw(); struct cfset_call_on_cpu_parm *p = parm; int set, set_size; size_t space; @@ -1348,9 +1565,9 @@ static void cfset_cpu_read(void *parm) cpuhw->used += space; cpuhw->sets += 1; } + debug_sprintf_event(cf_dbg, 4, "%s sets %d used %zd\n", __func__, + cpuhw->sets, cpuhw->used); } - debug_sprintf_event(cf_dbg, 4, "%s sets %d used %zd\n", __func__, - cpuhw->sets, cpuhw->used); } static int cfset_all_read(unsigned long arg, struct cfset_request *req) @@ -1513,7 +1730,6 @@ static int cfset_online_cpu(unsigned int cpu) struct cfset_call_on_cpu_parm p; struct cfset_request *rp; - mutex_lock(&cfset_ctrset_mutex); if (!list_empty(&cfset_session.head)) { list_for_each_entry(rp, &cfset_session.head, node) { p.sets = rp->ctrset; @@ -1521,19 +1737,18 @@ static int cfset_online_cpu(unsigned int cpu) cpumask_set_cpu(cpu, &rp->mask); } } - mutex_unlock(&cfset_ctrset_mutex); return 0; } /* Hotplug remove of a CPU. Scan through all active processes and clear * that CPU from the list of CPUs supplied with ioctl(..., START, ...). + * Adjust reference counts. */ static int cfset_offline_cpu(unsigned int cpu) { struct cfset_call_on_cpu_parm p; struct cfset_request *rp; - mutex_lock(&cfset_ctrset_mutex); if (!list_empty(&cfset_session.head)) { list_for_each_entry(rp, &cfset_session.head, node) { p.sets = rp->ctrset; @@ -1541,7 +1756,6 @@ static int cfset_offline_cpu(unsigned int cpu) cpumask_clear_cpu(cpu, &rp->mask); } } - mutex_unlock(&cfset_ctrset_mutex); return 0; } @@ -1619,7 +1833,8 @@ static int cfdiag_event_init(struct perf_event *event) } /* Initialize for using the CPU-measurement counter facility */ - cpumf_hw_inuse(); + if (cpum_cf_alloc(event->cpu)) + return -ENOMEM; event->destroy = hw_perf_event_destroy; err = cfdiag_event_init2(event); -- cgit v1.2.3