summaryrefslogtreecommitdiff
path: root/drivers/perf
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2022-03-25 20:11:38 +0300
committerLinus Torvalds <torvalds@linux-foundation.org>2022-03-25 20:11:38 +0300
commitaa5b537b0ecc16992577b013f11112d54c7ce869 (patch)
treeac9f6ce6c8c5b4722501cb36e95b3c0a35aa933e /drivers/perf
parentd710d370c4911e83da5d2bc43d4a2c3b56bd27e7 (diff)
parentbbde015227e89f1da21bd3b84523d62c4a445c06 (diff)
downloadlinux-aa5b537b0ecc16992577b013f11112d54c7ce869.tar.xz
Merge tag 'riscv-for-linus-5.18-mw0' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux
Pull RISC-V updates from Palmer Dabbelt: - Support for Sv57-based virtual memory. - Various improvements for the MicroChip PolarFire SOC and the associated Icicle dev board, which should allow upstream kernels to boot without any additional modifications. - An improved memmove() implementation. - Support for the new Ssconfpmf and SBI PMU extensions, which allows for a much more useful perf implementation on RISC-V systems. - Support for restartable sequences. * tag 'riscv-for-linus-5.18-mw0' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux: (36 commits) rseq/selftests: Add support for RISC-V RISC-V: Add support for restartable sequence MAINTAINERS: Add entry for RISC-V PMU drivers Documentation: riscv: Remove the old documentation RISC-V: Add sscofpmf extension support RISC-V: Add perf platform driver based on SBI PMU extension RISC-V: Add RISC-V SBI PMU extension definitions RISC-V: Add a simple platform driver for RISC-V legacy perf RISC-V: Add a perf core library for pmu drivers RISC-V: Add CSR encodings for all HPMCOUNTERS RISC-V: Remove the current perf implementation RISC-V: Improve /proc/cpuinfo output for ISA extensions RISC-V: Do no continue isa string parsing without correct XLEN RISC-V: Implement multi-letter ISA extension probing framework RISC-V: Extract multi-letter extension names from "riscv, isa" RISC-V: Minimal parser for "riscv, isa" strings RISC-V: Correctly print supported extensions riscv: Fixed misaligned memory access. Fixed pointer comparison. MAINTAINERS: update riscv/microchip entry riscv: dts: microchip: add new peripherals to icicle kit device tree ...
Diffstat (limited to 'drivers/perf')
-rw-r--r--drivers/perf/Kconfig30
-rw-r--r--drivers/perf/Makefile3
-rw-r--r--drivers/perf/riscv_pmu.c324
-rw-r--r--drivers/perf/riscv_pmu_legacy.c142
-rw-r--r--drivers/perf/riscv_pmu_sbi.c790
5 files changed, 1289 insertions, 0 deletions
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index d05ca6ebbb9d..afdcb91601d2 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -56,6 +56,36 @@ config ARM_PMU
Say y if you want to use CPU performance monitors on ARM-based
systems.
+config RISCV_PMU
+ depends on RISCV
+ bool "RISC-V PMU framework"
+ default y
+ help
+ Say y if you want to use CPU performance monitors on RISCV-based
+ systems. This provides the core PMU framework that abstracts common
+ PMU functionalities in a core library so that different PMU drivers
+ can reuse it.
+
+config RISCV_PMU_LEGACY
+ depends on RISCV_PMU
+ bool "RISC-V legacy PMU implementation"
+ default y
+ help
+ Say y if you want to use the legacy CPU performance monitor
+ implementation on RISC-V based systems. This only allows counting
+ of cycle/instruction counter and doesn't support counter overflow,
+ or programmable counters. It will be removed in future.
+
+config RISCV_PMU_SBI
+ depends on RISCV_PMU && RISCV_SBI
+ bool "RISC-V PMU based on SBI PMU extension"
+ default y
+ help
+ Say y if you want to use the CPU performance monitor
+ using SBI PMU extension on RISC-V based systems. This option provides
+ full perf feature support i.e. counter overflow, privilege mode
+ filtering, counter configuration.
+
config ARM_PMU_ACPI
depends on ARM_PMU && ACPI
def_bool y
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index 4f43080ec54e..57a279c61df5 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -10,6 +10,9 @@ obj-$(CONFIG_FSL_IMX8_DDR_PMU) += fsl_imx8_ddr_perf.o
obj-$(CONFIG_HISI_PMU) += hisilicon/
obj-$(CONFIG_QCOM_L2_PMU) += qcom_l2_pmu.o
obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
+obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
+obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
+obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o
diff --git a/drivers/perf/riscv_pmu.c b/drivers/perf/riscv_pmu.c
new file mode 100644
index 000000000000..b2b8d2074ed0
--- /dev/null
+++ b/drivers/perf/riscv_pmu.c
@@ -0,0 +1,324 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * RISC-V performance counter support.
+ *
+ * Copyright (C) 2021 Western Digital Corporation or its affiliates.
+ *
+ * This implementation is based on old RISC-V perf and ARM perf event code
+ * which are in turn based on sparc64 and x86 code.
+ */
+
+#include <linux/cpumask.h>
+#include <linux/irq.h>
+#include <linux/irqdesc.h>
+#include <linux/perf/riscv_pmu.h>
+#include <linux/printk.h>
+#include <linux/smp.h>
+
+#include <asm/sbi.h>
+
+static unsigned long csr_read_num(int csr_num)
+{
+#define switchcase_csr_read(__csr_num, __val) {\
+ case __csr_num: \
+ __val = csr_read(__csr_num); \
+ break; }
+#define switchcase_csr_read_2(__csr_num, __val) {\
+ switchcase_csr_read(__csr_num + 0, __val) \
+ switchcase_csr_read(__csr_num + 1, __val)}
+#define switchcase_csr_read_4(__csr_num, __val) {\
+ switchcase_csr_read_2(__csr_num + 0, __val) \
+ switchcase_csr_read_2(__csr_num + 2, __val)}
+#define switchcase_csr_read_8(__csr_num, __val) {\
+ switchcase_csr_read_4(__csr_num + 0, __val) \
+ switchcase_csr_read_4(__csr_num + 4, __val)}
+#define switchcase_csr_read_16(__csr_num, __val) {\
+ switchcase_csr_read_8(__csr_num + 0, __val) \
+ switchcase_csr_read_8(__csr_num + 8, __val)}
+#define switchcase_csr_read_32(__csr_num, __val) {\
+ switchcase_csr_read_16(__csr_num + 0, __val) \
+ switchcase_csr_read_16(__csr_num + 16, __val)}
+
+ unsigned long ret = 0;
+
+ switch (csr_num) {
+ switchcase_csr_read_32(CSR_CYCLE, ret)
+ switchcase_csr_read_32(CSR_CYCLEH, ret)
+ default :
+ break;
+ }
+
+ return ret;
+#undef switchcase_csr_read_32
+#undef switchcase_csr_read_16
+#undef switchcase_csr_read_8
+#undef switchcase_csr_read_4
+#undef switchcase_csr_read_2
+#undef switchcase_csr_read
+}
+
+/*
+ * Read the CSR of a corresponding counter.
+ */
+unsigned long riscv_pmu_ctr_read_csr(unsigned long csr)
+{
+ if (csr < CSR_CYCLE || csr > CSR_HPMCOUNTER31H ||
+ (csr > CSR_HPMCOUNTER31 && csr < CSR_CYCLEH)) {
+ pr_err("Invalid performance counter csr %lx\n", csr);
+ return -EINVAL;
+ }
+
+ return csr_read_num(csr);
+}
+
+u64 riscv_pmu_ctr_get_width_mask(struct perf_event *event)
+{
+ int cwidth;
+ struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+
+ if (!rvpmu->ctr_get_width)
+ /**
+ * If the pmu driver doesn't support counter width, set it to default
+ * maximum allowed by the specification.
+ */
+ cwidth = 63;
+ else {
+ if (hwc->idx == -1)
+ /* Handle init case where idx is not initialized yet */
+ cwidth = rvpmu->ctr_get_width(0);
+ else
+ cwidth = rvpmu->ctr_get_width(hwc->idx);
+ }
+
+ return GENMASK_ULL(cwidth, 0);
+}
+
+u64 riscv_pmu_event_update(struct perf_event *event)
+{
+ struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+ u64 prev_raw_count, new_raw_count;
+ unsigned long cmask;
+ u64 oldval, delta;
+
+ if (!rvpmu->ctr_read)
+ return 0;
+
+ cmask = riscv_pmu_ctr_get_width_mask(event);
+
+ do {
+ prev_raw_count = local64_read(&hwc->prev_count);
+ new_raw_count = rvpmu->ctr_read(event);
+ oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count,
+ new_raw_count);
+ } while (oldval != prev_raw_count);
+
+ delta = (new_raw_count - prev_raw_count) & cmask;
+ local64_add(delta, &event->count);
+ local64_sub(delta, &hwc->period_left);
+
+ return delta;
+}
+
+static void riscv_pmu_stop(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
+
+ WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
+
+ if (!(hwc->state & PERF_HES_STOPPED)) {
+ if (rvpmu->ctr_stop) {
+ rvpmu->ctr_stop(event, 0);
+ hwc->state |= PERF_HES_STOPPED;
+ }
+ riscv_pmu_event_update(event);
+ hwc->state |= PERF_HES_UPTODATE;
+ }
+}
+
+int riscv_pmu_event_set_period(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ s64 left = local64_read(&hwc->period_left);
+ s64 period = hwc->sample_period;
+ int overflow = 0;
+ uint64_t max_period = riscv_pmu_ctr_get_width_mask(event);
+
+ if (unlikely(left <= -period)) {
+ left = period;
+ local64_set(&hwc->period_left, left);
+ hwc->last_period = period;
+ overflow = 1;
+ }
+
+ if (unlikely(left <= 0)) {
+ left += period;
+ local64_set(&hwc->period_left, left);
+ hwc->last_period = period;
+ overflow = 1;
+ }
+
+ /*
+ * Limit the maximum period to prevent the counter value
+ * from overtaking the one we are about to program. In
+ * effect we are reducing max_period to account for
+ * interrupt latency (and we are being very conservative).
+ */
+ if (left > (max_period >> 1))
+ left = (max_period >> 1);
+
+ local64_set(&hwc->prev_count, (u64)-left);
+ perf_event_update_userpage(event);
+
+ return overflow;
+}
+
+static void riscv_pmu_start(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
+ uint64_t max_period = riscv_pmu_ctr_get_width_mask(event);
+ u64 init_val;
+
+ if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
+ return;
+
+ if (flags & PERF_EF_RELOAD)
+ WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
+
+ hwc->state = 0;
+ riscv_pmu_event_set_period(event);
+ init_val = local64_read(&hwc->prev_count) & max_period;
+ rvpmu->ctr_start(event, init_val);
+ perf_event_update_userpage(event);
+}
+
+static int riscv_pmu_add(struct perf_event *event, int flags)
+{
+ struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
+ struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
+ struct hw_perf_event *hwc = &event->hw;
+ int idx;
+
+ idx = rvpmu->ctr_get_idx(event);
+ if (idx < 0)
+ return idx;
+
+ hwc->idx = idx;
+ cpuc->events[idx] = event;
+ cpuc->n_events++;
+ hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
+ if (flags & PERF_EF_START)
+ riscv_pmu_start(event, PERF_EF_RELOAD);
+
+ /* Propagate our changes to the userspace mapping. */
+ perf_event_update_userpage(event);
+
+ return 0;
+}
+
+static void riscv_pmu_del(struct perf_event *event, int flags)
+{
+ struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
+ struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
+ struct hw_perf_event *hwc = &event->hw;
+
+ riscv_pmu_stop(event, PERF_EF_UPDATE);
+ cpuc->events[hwc->idx] = NULL;
+ /* The firmware need to reset the counter mapping */
+ if (rvpmu->ctr_stop)
+ rvpmu->ctr_stop(event, RISCV_PMU_STOP_FLAG_RESET);
+ cpuc->n_events--;
+ if (rvpmu->ctr_clear_idx)
+ rvpmu->ctr_clear_idx(event);
+ perf_event_update_userpage(event);
+ hwc->idx = -1;
+}
+
+static void riscv_pmu_read(struct perf_event *event)
+{
+ riscv_pmu_event_update(event);
+}
+
+static int riscv_pmu_event_init(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
+ int mapped_event;
+ u64 event_config = 0;
+ uint64_t cmask;
+
+ hwc->flags = 0;
+ mapped_event = rvpmu->event_map(event, &event_config);
+ if (mapped_event < 0) {
+ pr_debug("event %x:%llx not supported\n", event->attr.type,
+ event->attr.config);
+ return mapped_event;
+ }
+
+ /*
+ * idx is set to -1 because the index of a general event should not be
+ * decided until binding to some counter in pmu->add().
+ * config will contain the information about counter CSR
+ * the idx will contain the counter index
+ */
+ hwc->config = event_config;
+ hwc->idx = -1;
+ hwc->event_base = mapped_event;
+
+ if (!is_sampling_event(event)) {
+ /*
+ * For non-sampling runs, limit the sample_period to half
+ * of the counter width. That way, the new counter value
+ * is far less likely to overtake the previous one unless
+ * you have some serious IRQ latency issues.
+ */
+ cmask = riscv_pmu_ctr_get_width_mask(event);
+ hwc->sample_period = cmask >> 1;
+ hwc->last_period = hwc->sample_period;
+ local64_set(&hwc->period_left, hwc->sample_period);
+ }
+
+ return 0;
+}
+
+struct riscv_pmu *riscv_pmu_alloc(void)
+{
+ struct riscv_pmu *pmu;
+ int cpuid, i;
+ struct cpu_hw_events *cpuc;
+
+ pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
+ if (!pmu)
+ goto out;
+
+ pmu->hw_events = alloc_percpu_gfp(struct cpu_hw_events, GFP_KERNEL);
+ if (!pmu->hw_events) {
+ pr_info("failed to allocate per-cpu PMU data.\n");
+ goto out_free_pmu;
+ }
+
+ for_each_possible_cpu(cpuid) {
+ cpuc = per_cpu_ptr(pmu->hw_events, cpuid);
+ cpuc->n_events = 0;
+ for (i = 0; i < RISCV_MAX_COUNTERS; i++)
+ cpuc->events[i] = NULL;
+ }
+ pmu->pmu = (struct pmu) {
+ .event_init = riscv_pmu_event_init,
+ .add = riscv_pmu_add,
+ .del = riscv_pmu_del,
+ .start = riscv_pmu_start,
+ .stop = riscv_pmu_stop,
+ .read = riscv_pmu_read,
+ };
+
+ return pmu;
+
+out_free_pmu:
+ kfree(pmu);
+out:
+ return NULL;
+}
diff --git a/drivers/perf/riscv_pmu_legacy.c b/drivers/perf/riscv_pmu_legacy.c
new file mode 100644
index 000000000000..342778782359
--- /dev/null
+++ b/drivers/perf/riscv_pmu_legacy.c
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * RISC-V performance counter support.
+ *
+ * Copyright (C) 2021 Western Digital Corporation or its affiliates.
+ *
+ * This implementation is based on old RISC-V perf and ARM perf event code
+ * which are in turn based on sparc64 and x86 code.
+ */
+
+#include <linux/mod_devicetable.h>
+#include <linux/perf/riscv_pmu.h>
+#include <linux/platform_device.h>
+
+#define RISCV_PMU_LEGACY_CYCLE 0
+#define RISCV_PMU_LEGACY_INSTRET 1
+#define RISCV_PMU_LEGACY_NUM_CTR 2
+
+static bool pmu_init_done;
+
+static int pmu_legacy_ctr_get_idx(struct perf_event *event)
+{
+ struct perf_event_attr *attr = &event->attr;
+
+ if (event->attr.type != PERF_TYPE_HARDWARE)
+ return -EOPNOTSUPP;
+ if (attr->config == PERF_COUNT_HW_CPU_CYCLES)
+ return RISCV_PMU_LEGACY_CYCLE;
+ else if (attr->config == PERF_COUNT_HW_INSTRUCTIONS)
+ return RISCV_PMU_LEGACY_INSTRET;
+ else
+ return -EOPNOTSUPP;
+}
+
+/* For legacy config & counter index are same */
+static int pmu_legacy_event_map(struct perf_event *event, u64 *config)
+{
+ return pmu_legacy_ctr_get_idx(event);
+}
+
+static u64 pmu_legacy_read_ctr(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ int idx = hwc->idx;
+ u64 val;
+
+ if (idx == RISCV_PMU_LEGACY_CYCLE) {
+ val = riscv_pmu_ctr_read_csr(CSR_CYCLE);
+ if (IS_ENABLED(CONFIG_32BIT))
+ val = (u64)riscv_pmu_ctr_read_csr(CSR_CYCLEH) << 32 | val;
+ } else if (idx == RISCV_PMU_LEGACY_INSTRET) {
+ val = riscv_pmu_ctr_read_csr(CSR_INSTRET);
+ if (IS_ENABLED(CONFIG_32BIT))
+ val = ((u64)riscv_pmu_ctr_read_csr(CSR_INSTRETH)) << 32 | val;
+ } else
+ return 0;
+
+ return val;
+}
+
+static void pmu_legacy_ctr_start(struct perf_event *event, u64 ival)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ u64 initial_val = pmu_legacy_read_ctr(event);
+
+ /**
+ * The legacy method doesn't really have a start/stop method.
+ * It also can not update the counter with a initial value.
+ * But we still need to set the prev_count so that read() can compute
+ * the delta. Just use the current counter value to set the prev_count.
+ */
+ local64_set(&hwc->prev_count, initial_val);
+}
+
+/**
+ * This is just a simple implementation to allow legacy implementations
+ * compatible with new RISC-V PMU driver framework.
+ * This driver only allows reading two counters i.e CYCLE & INSTRET.
+ * However, it can not start or stop the counter. Thus, it is not very useful
+ * will be removed in future.
+ */
+static void pmu_legacy_init(struct riscv_pmu *pmu)
+{
+ pr_info("Legacy PMU implementation is available\n");
+
+ pmu->num_counters = RISCV_PMU_LEGACY_NUM_CTR;
+ pmu->ctr_start = pmu_legacy_ctr_start;
+ pmu->ctr_stop = NULL;
+ pmu->event_map = pmu_legacy_event_map;
+ pmu->ctr_get_idx = pmu_legacy_ctr_get_idx;
+ pmu->ctr_get_width = NULL;
+ pmu->ctr_clear_idx = NULL;
+ pmu->ctr_read = pmu_legacy_read_ctr;
+
+ perf_pmu_register(&pmu->pmu, "cpu", PERF_TYPE_RAW);
+}
+
+static int pmu_legacy_device_probe(struct platform_device *pdev)
+{
+ struct riscv_pmu *pmu = NULL;
+
+ pmu = riscv_pmu_alloc();
+ if (!pmu)
+ return -ENOMEM;
+ pmu_legacy_init(pmu);
+
+ return 0;
+}
+
+static struct platform_driver pmu_legacy_driver = {
+ .probe = pmu_legacy_device_probe,
+ .driver = {
+ .name = RISCV_PMU_LEGACY_PDEV_NAME,
+ },
+};
+
+static int __init riscv_pmu_legacy_devinit(void)
+{
+ int ret;
+ struct platform_device *pdev;
+
+ if (likely(pmu_init_done))
+ return 0;
+
+ ret = platform_driver_register(&pmu_legacy_driver);
+ if (ret)
+ return ret;
+
+ pdev = platform_device_register_simple(RISCV_PMU_LEGACY_PDEV_NAME, -1, NULL, 0);
+ if (IS_ERR(pdev)) {
+ platform_driver_unregister(&pmu_legacy_driver);
+ return PTR_ERR(pdev);
+ }
+
+ return ret;
+}
+late_initcall(riscv_pmu_legacy_devinit);
+
+void riscv_pmu_legacy_skip_init(void)
+{
+ pmu_init_done = true;
+}
diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c
new file mode 100644
index 000000000000..a1317a483512
--- /dev/null
+++ b/drivers/perf/riscv_pmu_sbi.c
@@ -0,0 +1,790 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * RISC-V performance counter support.
+ *
+ * Copyright (C) 2021 Western Digital Corporation or its affiliates.
+ *
+ * This code is based on ARM perf event code which is in turn based on
+ * sparc64 and x86 code.
+ */
+
+#define pr_fmt(fmt) "riscv-pmu-sbi: " fmt
+
+#include <linux/mod_devicetable.h>
+#include <linux/perf/riscv_pmu.h>
+#include <linux/platform_device.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of_irq.h>
+#include <linux/of.h>
+
+#include <asm/sbi.h>
+#include <asm/hwcap.h>
+
+union sbi_pmu_ctr_info {
+ unsigned long value;
+ struct {
+ unsigned long csr:12;
+ unsigned long width:6;
+#if __riscv_xlen == 32
+ unsigned long reserved:13;
+#else
+ unsigned long reserved:45;
+#endif
+ unsigned long type:1;
+ };
+};
+
+/**
+ * RISC-V doesn't have hetergenous harts yet. This need to be part of
+ * per_cpu in case of harts with different pmu counters
+ */
+static union sbi_pmu_ctr_info *pmu_ctr_list;
+static unsigned int riscv_pmu_irq;
+
+struct sbi_pmu_event_data {
+ union {
+ union {
+ struct hw_gen_event {
+ uint32_t event_code:16;
+ uint32_t event_type:4;
+ uint32_t reserved:12;
+ } hw_gen_event;
+ struct hw_cache_event {
+ uint32_t result_id:1;
+ uint32_t op_id:2;
+ uint32_t cache_id:13;
+ uint32_t event_type:4;
+ uint32_t reserved:12;
+ } hw_cache_event;
+ };
+ uint32_t event_idx;
+ };
+};
+
+static const struct sbi_pmu_event_data pmu_hw_event_map[] = {
+ [PERF_COUNT_HW_CPU_CYCLES] = {.hw_gen_event = {
+ SBI_PMU_HW_CPU_CYCLES,
+ SBI_PMU_EVENT_TYPE_HW, 0}},
+ [PERF_COUNT_HW_INSTRUCTIONS] = {.hw_gen_event = {
+ SBI_PMU_HW_INSTRUCTIONS,
+ SBI_PMU_EVENT_TYPE_HW, 0}},
+ [PERF_COUNT_HW_CACHE_REFERENCES] = {.hw_gen_event = {
+ SBI_PMU_HW_CACHE_REFERENCES,
+ SBI_PMU_EVENT_TYPE_HW, 0}},
+ [PERF_COUNT_HW_CACHE_MISSES] = {.hw_gen_event = {
+ SBI_PMU_HW_CACHE_MISSES,
+ SBI_PMU_EVENT_TYPE_HW, 0}},
+ [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = {.hw_gen_event = {
+ SBI_PMU_HW_BRANCH_INSTRUCTIONS,
+ SBI_PMU_EVENT_TYPE_HW, 0}},
+ [PERF_COUNT_HW_BRANCH_MISSES] = {.hw_gen_event = {
+ SBI_PMU_HW_BRANCH_MISSES,
+ SBI_PMU_EVENT_TYPE_HW, 0}},
+ [PERF_COUNT_HW_BUS_CYCLES] = {.hw_gen_event = {
+ SBI_PMU_HW_BUS_CYCLES,
+ SBI_PMU_EVENT_TYPE_HW, 0}},
+ [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = {.hw_gen_event = {
+ SBI_PMU_HW_STALLED_CYCLES_FRONTEND,
+ SBI_PMU_EVENT_TYPE_HW, 0}},
+ [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = {.hw_gen_event = {
+ SBI_PMU_HW_STALLED_CYCLES_BACKEND,
+ SBI_PMU_EVENT_TYPE_HW, 0}},
+ [PERF_COUNT_HW_REF_CPU_CYCLES] = {.hw_gen_event = {
+ SBI_PMU_HW_REF_CPU_CYCLES,
+ SBI_PMU_EVENT_TYPE_HW, 0}},
+};
+
+#define C(x) PERF_COUNT_HW_CACHE_##x
+static const struct sbi_pmu_event_data pmu_cache_event_map[PERF_COUNT_HW_CACHE_MAX]
+[PERF_COUNT_HW_CACHE_OP_MAX]
+[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+ [C(L1D)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ },
+ [C(L1I)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_READ), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), C(OP_READ),
+ C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ },
+ [C(LL)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ },
+ [C(DTLB)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ },
+ [C(ITLB)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ },
+ [C(BPU)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ },
+ [C(NODE)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
+ C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
+ C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
+ },
+ },
+};
+
+static int pmu_sbi_ctr_get_width(int idx)
+{
+ return pmu_ctr_list[idx].width;
+}
+
+static bool pmu_sbi_ctr_is_fw(int cidx)
+{
+ union sbi_pmu_ctr_info *info;
+
+ info = &pmu_ctr_list[cidx];
+ if (!info)
+ return false;
+
+ return (info->type == SBI_PMU_CTR_TYPE_FW) ? true : false;
+}
+
+static int pmu_sbi_ctr_get_idx(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
+ struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
+ struct sbiret ret;
+ int idx;
+ uint64_t cbase = 0;
+ uint64_t cmask = GENMASK_ULL(rvpmu->num_counters - 1, 0);
+ unsigned long cflags = 0;
+
+ if (event->attr.exclude_kernel)
+ cflags |= SBI_PMU_CFG_FLAG_SET_SINH;
+ if (event->attr.exclude_user)
+ cflags |= SBI_PMU_CFG_FLAG_SET_UINH;
+
+ /* retrieve the available counter index */
+ ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, cmask,
+ cflags, hwc->event_base, hwc->config, 0);
+ if (ret.error) {
+ pr_debug("Not able to find a counter for event %lx config %llx\n",
+ hwc->event_base, hwc->config);
+ return sbi_err_map_linux_errno(ret.error);
+ }
+
+ idx = ret.value;
+ if (idx >= rvpmu->num_counters || !pmu_ctr_list[idx].value)
+ return -ENOENT;
+
+ /* Additional sanity check for the counter id */
+ if (pmu_sbi_ctr_is_fw(idx)) {
+ if (!test_and_set_bit(idx, cpuc->used_fw_ctrs))
+ return idx;
+ } else {
+ if (!test_and_set_bit(idx, cpuc->used_hw_ctrs))
+ return idx;
+ }
+
+ return -ENOENT;
+}
+
+static void pmu_sbi_ctr_clear_idx(struct perf_event *event)
+{
+
+ struct hw_perf_event *hwc = &event->hw;
+ struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
+ struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
+ int idx = hwc->idx;
+
+ if (pmu_sbi_ctr_is_fw(idx))
+ clear_bit(idx, cpuc->used_fw_ctrs);
+ else
+ clear_bit(idx, cpuc->used_hw_ctrs);
+}
+
+static int pmu_event_find_cache(u64 config)
+{
+ unsigned int cache_type, cache_op, cache_result, ret;
+
+ cache_type = (config >> 0) & 0xff;
+ if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
+ return -EINVAL;
+
+ cache_op = (config >> 8) & 0xff;
+ if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
+ return -EINVAL;
+
+ cache_result = (config >> 16) & 0xff;
+ if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
+ return -EINVAL;
+
+ ret = pmu_cache_event_map[cache_type][cache_op][cache_result].event_idx;
+
+ return ret;
+}
+
+static bool pmu_sbi_is_fw_event(struct perf_event *event)
+{
+ u32 type = event->attr.type;
+ u64 config = event->attr.config;
+
+ if ((type == PERF_TYPE_RAW) && ((config >> 63) == 1))
+ return true;
+ else
+ return false;
+}
+
+static int pmu_sbi_event_map(struct perf_event *event, u64 *econfig)
+{
+ u32 type = event->attr.type;
+ u64 config = event->attr.config;
+ int bSoftware;
+ u64 raw_config_val;
+ int ret;
+
+ switch (type) {
+ case PERF_TYPE_HARDWARE:
+ if (config >= PERF_COUNT_HW_MAX)
+ return -EINVAL;
+ ret = pmu_hw_event_map[event->attr.config].event_idx;
+ break;
+ case PERF_TYPE_HW_CACHE:
+ ret = pmu_event_find_cache(config);
+ break;
+ case PERF_TYPE_RAW:
+ /*
+ * As per SBI specification, the upper 16 bits must be unused for
+ * a raw event. Use the MSB (63b) to distinguish between hardware
+ * raw event and firmware events.
+ */
+ bSoftware = config >> 63;
+ raw_config_val = config & RISCV_PMU_RAW_EVENT_MASK;
+ if (bSoftware) {
+ if (raw_config_val < SBI_PMU_FW_MAX)
+ ret = (raw_config_val & 0xFFFF) |
+ (SBI_PMU_EVENT_TYPE_FW << 16);
+ else
+ return -EINVAL;
+ } else {
+ ret = RISCV_PMU_RAW_EVENT_IDX;
+ *econfig = raw_config_val;
+ }
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ return ret;
+}
+
+static u64 pmu_sbi_ctr_read(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ int idx = hwc->idx;
+ struct sbiret ret;
+ union sbi_pmu_ctr_info info;
+ u64 val = 0;
+
+ if (pmu_sbi_is_fw_event(event)) {
+ ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ,
+ hwc->idx, 0, 0, 0, 0, 0);
+ if (!ret.error)
+ val = ret.value;
+ } else {
+ info = pmu_ctr_list[idx];
+ val = riscv_pmu_ctr_read_csr(info.csr);
+ if (IS_ENABLED(CONFIG_32BIT))
+ val = ((u64)riscv_pmu_ctr_read_csr(info.csr + 0x80)) << 31 | val;
+ }
+
+ return val;
+}
+
+static void pmu_sbi_ctr_start(struct perf_event *event, u64 ival)
+{
+ struct sbiret ret;
+ struct hw_perf_event *hwc = &event->hw;
+ unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE;
+
+ ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx,
+ 1, flag, ival, ival >> 32, 0);
+ if (ret.error && (ret.error != SBI_ERR_ALREADY_STARTED))
+ pr_err("Starting counter idx %d failed with error %d\n",
+ hwc->idx, sbi_err_map_linux_errno(ret.error));
+}
+
+static void pmu_sbi_ctr_stop(struct perf_event *event, unsigned long flag)
+{
+ struct sbiret ret;
+ struct hw_perf_event *hwc = &event->hw;
+
+ ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, hwc->idx, 1, flag, 0, 0, 0);
+ if (ret.error && (ret.error != SBI_ERR_ALREADY_STOPPED) &&
+ flag != SBI_PMU_STOP_FLAG_RESET)
+ pr_err("Stopping counter idx %d failed with error %d\n",
+ hwc->idx, sbi_err_map_linux_errno(ret.error));
+}
+
+static int pmu_sbi_find_num_ctrs(void)
+{
+ struct sbiret ret;
+
+ ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_NUM_COUNTERS, 0, 0, 0, 0, 0, 0);
+ if (!ret.error)
+ return ret.value;
+ else
+ return sbi_err_map_linux_errno(ret.error);
+}
+
+static int pmu_sbi_get_ctrinfo(int nctr)
+{
+ struct sbiret ret;
+ int i, num_hw_ctr = 0, num_fw_ctr = 0;
+ union sbi_pmu_ctr_info cinfo;
+
+ pmu_ctr_list = kcalloc(nctr, sizeof(*pmu_ctr_list), GFP_KERNEL);
+ if (!pmu_ctr_list)
+ return -ENOMEM;
+
+ for (i = 0; i <= nctr; i++) {
+ ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_GET_INFO, i, 0, 0, 0, 0, 0);
+ if (ret.error)
+ /* The logical counter ids are not expected to be contiguous */
+ continue;
+ cinfo.value = ret.value;
+ if (cinfo.type == SBI_PMU_CTR_TYPE_FW)
+ num_fw_ctr++;
+ else
+ num_hw_ctr++;
+ pmu_ctr_list[i].value = cinfo.value;
+ }
+
+ pr_info("%d firmware and %d hardware counters\n", num_fw_ctr, num_hw_ctr);
+
+ return 0;
+}
+
+static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu)
+{
+ /**
+ * No need to check the error because we are disabling all the counters
+ * which may include counters that are not enabled yet.
+ */
+ sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP,
+ 0, GENMASK_ULL(pmu->num_counters - 1, 0), 0, 0, 0, 0);
+}
+
+static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu)
+{
+ struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
+
+ /* No need to check the error here as we can't do anything about the error */
+ sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, 0,
+ cpu_hw_evt->used_hw_ctrs[0], 0, 0, 0, 0);
+}
+
+/**
+ * This function starts all the used counters in two step approach.
+ * Any counter that did not overflow can be start in a single step
+ * while the overflowed counters need to be started with updated initialization
+ * value.
+ */
+static inline void pmu_sbi_start_overflow_mask(struct riscv_pmu *pmu,
+ unsigned long ctr_ovf_mask)
+{
+ int idx = 0;
+ struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
+ struct perf_event *event;
+ unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE;
+ unsigned long ctr_start_mask = 0;
+ uint64_t max_period;
+ struct hw_perf_event *hwc;
+ u64 init_val = 0;
+
+ ctr_start_mask = cpu_hw_evt->used_hw_ctrs[0] & ~ctr_ovf_mask;
+
+ /* Start all the counters that did not overflow in a single shot */
+ sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, 0, ctr_start_mask,
+ 0, 0, 0, 0);
+
+ /* Reinitialize and start all the counter that overflowed */
+ while (ctr_ovf_mask) {
+ if (ctr_ovf_mask & 0x01) {
+ event = cpu_hw_evt->events[idx];
+ hwc = &event->hw;
+ max_period = riscv_pmu_ctr_get_width_mask(event);
+ init_val = local64_read(&hwc->prev_count) & max_period;
+ sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1,
+ flag, init_val, 0, 0);
+ }
+ ctr_ovf_mask = ctr_ovf_mask >> 1;
+ idx++;
+ }
+}
+
+static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev)
+{
+ struct perf_sample_data data;
+ struct pt_regs *regs;
+ struct hw_perf_event *hw_evt;
+ union sbi_pmu_ctr_info *info;
+ int lidx, hidx, fidx;
+ struct riscv_pmu *pmu;
+ struct perf_event *event;
+ unsigned long overflow;
+ unsigned long overflowed_ctrs = 0;
+ struct cpu_hw_events *cpu_hw_evt = dev;
+
+ if (WARN_ON_ONCE(!cpu_hw_evt))
+ return IRQ_NONE;
+
+ /* Firmware counter don't support overflow yet */
+ fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS);
+ event = cpu_hw_evt->events[fidx];
+ if (!event) {
+ csr_clear(CSR_SIP, SIP_LCOFIP);
+ return IRQ_NONE;
+ }
+
+ pmu = to_riscv_pmu(event->pmu);
+ pmu_sbi_stop_hw_ctrs(pmu);
+
+ /* Overflow status register should only be read after counter are stopped */
+ overflow = csr_read(CSR_SSCOUNTOVF);
+
+ /**
+ * Overflow interrupt pending bit should only be cleared after stopping
+ * all the counters to avoid any race condition.
+ */
+ csr_clear(CSR_SIP, SIP_LCOFIP);
+
+ /* No overflow bit is set */
+ if (!overflow)
+ return IRQ_NONE;
+
+ regs = get_irq_regs();
+
+ for_each_set_bit(lidx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) {
+ struct perf_event *event = cpu_hw_evt->events[lidx];
+
+ /* Skip if invalid event or user did not request a sampling */
+ if (!event || !is_sampling_event(event))
+ continue;
+
+ info = &pmu_ctr_list[lidx];
+ /* Do a sanity check */
+ if (!info || info->type != SBI_PMU_CTR_TYPE_HW)
+ continue;
+
+ /* compute hardware counter index */
+ hidx = info->csr - CSR_CYCLE;
+ /* check if the corresponding bit is set in sscountovf */
+ if (!(overflow & (1 << hidx)))
+ continue;
+
+ /*
+ * Keep a track of overflowed counters so that they can be started
+ * with updated initial value.
+ */
+ overflowed_ctrs |= 1 << lidx;
+ hw_evt = &event->hw;
+ riscv_pmu_event_update(event);
+ perf_sample_data_init(&data, 0, hw_evt->last_period);
+ if (riscv_pmu_event_set_period(event)) {
+ /*
+ * Unlike other ISAs, RISC-V don't have to disable interrupts
+ * to avoid throttling here. As per the specification, the
+ * interrupt remains disabled until the OF bit is set.
+ * Interrupts are enabled again only during the start.
+ * TODO: We will need to stop the guest counters once
+ * virtualization support is added.
+ */
+ perf_event_overflow(event, &data, regs);
+ }
+ }
+ pmu_sbi_start_overflow_mask(pmu, overflowed_ctrs);
+
+ return IRQ_HANDLED;
+}
+
+static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node)
+{
+ struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node);
+ struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
+
+ /* Enable the access for TIME csr only from the user mode now */
+ csr_write(CSR_SCOUNTEREN, 0x2);
+
+ /* Stop all the counters so that they can be enabled from perf */
+ pmu_sbi_stop_all(pmu);
+
+ if (riscv_isa_extension_available(NULL, SSCOFPMF)) {
+ cpu_hw_evt->irq = riscv_pmu_irq;
+ csr_clear(CSR_IP, BIT(RV_IRQ_PMU));
+ csr_set(CSR_IE, BIT(RV_IRQ_PMU));
+ enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE);
+ }
+
+ return 0;
+}
+
+static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node)
+{
+ if (riscv_isa_extension_available(NULL, SSCOFPMF)) {
+ disable_percpu_irq(riscv_pmu_irq);
+ csr_clear(CSR_IE, BIT(RV_IRQ_PMU));
+ }
+
+ /* Disable all counters access for user mode now */
+ csr_write(CSR_SCOUNTEREN, 0x0);
+
+ return 0;
+}
+
+static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pdev)
+{
+ int ret;
+ struct cpu_hw_events __percpu *hw_events = pmu->hw_events;
+ struct device_node *cpu, *child;
+ struct irq_domain *domain = NULL;
+
+ if (!riscv_isa_extension_available(NULL, SSCOFPMF))
+ return -EOPNOTSUPP;
+
+ for_each_of_cpu_node(cpu) {
+ child = of_get_compatible_child(cpu, "riscv,cpu-intc");
+ if (!child) {
+ pr_err("Failed to find INTC node\n");
+ return -ENODEV;
+ }
+ domain = irq_find_host(child);
+ of_node_put(child);
+ if (domain)
+ break;
+ }
+ if (!domain) {
+ pr_err("Failed to find INTC IRQ root domain\n");
+ return -ENODEV;
+ }
+
+ riscv_pmu_irq = irq_create_mapping(domain, RV_IRQ_PMU);
+ if (!riscv_pmu_irq) {
+ pr_err("Failed to map PMU interrupt for node\n");
+ return -ENODEV;
+ }
+
+ ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events);
+ if (ret) {
+ pr_err("registering percpu irq failed [%d]\n", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static int pmu_sbi_device_probe(struct platform_device *pdev)
+{
+ struct riscv_pmu *pmu = NULL;
+ int num_counters;
+ int ret = -ENODEV;
+
+ pr_info("SBI PMU extension is available\n");
+ pmu = riscv_pmu_alloc();
+ if (!pmu)
+ return -ENOMEM;
+
+ num_counters = pmu_sbi_find_num_ctrs();
+ if (num_counters < 0) {
+ pr_err("SBI PMU extension doesn't provide any counters\n");
+ goto out_free;
+ }
+
+ /* cache all the information about counters now */
+ if (pmu_sbi_get_ctrinfo(num_counters))
+ goto out_free;
+
+ ret = pmu_sbi_setup_irqs(pmu, pdev);
+ if (ret < 0) {
+ pr_info("Perf sampling/filtering is not supported as sscof extension is not available\n");
+ pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
+ pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE;
+ }
+ pmu->num_counters = num_counters;
+ pmu->ctr_start = pmu_sbi_ctr_start;
+ pmu->ctr_stop = pmu_sbi_ctr_stop;
+ pmu->event_map = pmu_sbi_event_map;
+ pmu->ctr_get_idx = pmu_sbi_ctr_get_idx;
+ pmu->ctr_get_width = pmu_sbi_ctr_get_width;
+ pmu->ctr_clear_idx = pmu_sbi_ctr_clear_idx;
+ pmu->ctr_read = pmu_sbi_ctr_read;
+
+ ret = cpuhp_state_add_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node);
+ if (ret)
+ return ret;
+
+ ret = perf_pmu_register(&pmu->pmu, "cpu", PERF_TYPE_RAW);
+ if (ret) {
+ cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node);
+ return ret;
+ }
+
+ return 0;
+
+out_free:
+ kfree(pmu);
+ return ret;
+}
+
+static struct platform_driver pmu_sbi_driver = {
+ .probe = pmu_sbi_device_probe,
+ .driver = {
+ .name = RISCV_PMU_PDEV_NAME,
+ },
+};
+
+static int __init pmu_sbi_devinit(void)
+{
+ int ret;
+ struct platform_device *pdev;
+
+ if (sbi_spec_version < sbi_mk_version(0, 3) ||
+ sbi_probe_extension(SBI_EXT_PMU) <= 0) {
+ return 0;
+ }
+
+ ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_RISCV_STARTING,
+ "perf/riscv/pmu:starting",
+ pmu_sbi_starting_cpu, pmu_sbi_dying_cpu);
+ if (ret) {
+ pr_err("CPU hotplug notifier could not be registered: %d\n",
+ ret);
+ return ret;
+ }
+
+ ret = platform_driver_register(&pmu_sbi_driver);
+ if (ret)
+ return ret;
+
+ pdev = platform_device_register_simple(RISCV_PMU_PDEV_NAME, -1, NULL, 0);
+ if (IS_ERR(pdev)) {
+ platform_driver_unregister(&pmu_sbi_driver);
+ return PTR_ERR(pdev);
+ }
+
+ /* Notify legacy implementation that SBI pmu is available*/
+ riscv_pmu_legacy_skip_init();
+
+ return ret;
+}
+device_initcall(pmu_sbi_devinit)