diff options
Diffstat (limited to 'drivers/cxl/core')
| -rw-r--r-- | drivers/cxl/core/Makefile | 1 | ||||
| -rw-r--r-- | drivers/cxl/core/core.h | 11 | ||||
| -rw-r--r-- | drivers/cxl/core/hdm.c | 48 | ||||
| -rw-r--r-- | drivers/cxl/core/mbox.c | 339 | ||||
| -rw-r--r-- | drivers/cxl/core/memdev.c | 503 | ||||
| -rw-r--r-- | drivers/cxl/core/pci.c | 31 | ||||
| -rw-r--r-- | drivers/cxl/core/pmem.c | 2 | ||||
| -rw-r--r-- | drivers/cxl/core/pmu.c | 68 | ||||
| -rw-r--r-- | drivers/cxl/core/port.c | 163 | ||||
| -rw-r--r-- | drivers/cxl/core/region.c | 168 | ||||
| -rw-r--r-- | drivers/cxl/core/regs.c | 182 |
11 files changed, 1188 insertions, 328 deletions
diff --git a/drivers/cxl/core/Makefile b/drivers/cxl/core/Makefile index ca4ae31d8f57..1f66b5d4d935 100644 --- a/drivers/cxl/core/Makefile +++ b/drivers/cxl/core/Makefile @@ -12,5 +12,6 @@ cxl_core-y += memdev.o cxl_core-y += mbox.o cxl_core-y += pci.o cxl_core-y += hdm.o +cxl_core-y += pmu.o cxl_core-$(CONFIG_TRACING) += trace.o cxl_core-$(CONFIG_CXL_REGION) += region.o diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 27f0968449de..45e7e044cf4a 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -6,6 +6,7 @@ extern const struct device_type cxl_nvdimm_bridge_type; extern const struct device_type cxl_nvdimm_type; +extern const struct device_type cxl_pmu_type; extern struct attribute_group cxl_base_attribute_group; @@ -63,6 +64,16 @@ int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size); int cxl_dpa_free(struct cxl_endpoint_decoder *cxled); resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled); resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled); + +enum cxl_rcrb { + CXL_RCRB_DOWNSTREAM, + CXL_RCRB_UPSTREAM, +}; +struct cxl_rcrb_info; +resource_size_t __rcrb_to_component(struct device *dev, + struct cxl_rcrb_info *ri, + enum cxl_rcrb which); + extern struct rw_semaphore cxl_dpa_rwsem; int cxl_memdev_init(void); diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index 7889ff203a34..4449b34a80cc 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -85,6 +85,7 @@ static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb, struct cxl_component_regs *regs) { struct cxl_register_map map = { + .dev = &port->dev, .resource = port->component_reg_phys, .base = crb, .max_size = CXL_COMPONENT_REG_BLOCK_SIZE, @@ -97,8 +98,7 @@ static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb, return -ENODEV; } - return cxl_map_component_regs(&port->dev, regs, &map, - BIT(CXL_CM_CAP_CAP_ID_HDM)); + return cxl_map_component_regs(&map, regs, BIT(CXL_CM_CAP_CAP_ID_HDM)); } static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) @@ -570,8 +570,9 @@ static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl) static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl) { - u32p_replace_bits(ctrl, !!(cxld->target_type == 3), - CXL_HDM_DECODER0_CTRL_TYPE); + u32p_replace_bits(ctrl, + !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM), + CXL_HDM_DECODER0_CTRL_HOSTONLY); } static int cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt) @@ -764,7 +765,7 @@ static int cxl_setup_hdm_decoder_from_dvsec( if (!len) return -ENOENT; - cxld->target_type = CXL_DECODER_EXPANDER; + cxld->target_type = CXL_DECODER_HOSTONLYMEM; cxld->commit = NULL; cxld->reset = NULL; cxld->hpa_range = info->dvsec_range[which]; @@ -793,8 +794,8 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, int *target_map, void __iomem *hdm, int which, u64 *dpa_base, struct cxl_endpoint_dvsec_info *info) { + struct cxl_endpoint_decoder *cxled = NULL; u64 size, base, skip, dpa_size, lo, hi; - struct cxl_endpoint_decoder *cxled; bool committed; u32 remainder; int i, rc; @@ -827,6 +828,8 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, return -ENXIO; } + if (info) + cxled = to_cxl_endpoint_decoder(&cxld->dev); cxld->hpa_range = (struct range) { .start = base, .end = base + size - 1, @@ -837,10 +840,10 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, cxld->flags |= CXL_DECODER_F_ENABLE; if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK) cxld->flags |= CXL_DECODER_F_LOCK; - if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl)) - cxld->target_type = CXL_DECODER_EXPANDER; + if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl)) + cxld->target_type = CXL_DECODER_HOSTONLYMEM; else - cxld->target_type = CXL_DECODER_ACCELERATOR; + cxld->target_type = CXL_DECODER_DEVMEM; if (cxld->id != port->commit_end + 1) { dev_warn(&port->dev, "decoder%d.%d: Committed out of order\n", @@ -856,12 +859,28 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, } port->commit_end = cxld->id; } else { - /* unless / until type-2 drivers arrive, assume type-3 */ - if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl) == 0) { - ctrl |= CXL_HDM_DECODER0_CTRL_TYPE; + if (cxled) { + struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + + /* + * Default by devtype until a device arrives that needs + * more precision. + */ + if (cxlds->type == CXL_DEVTYPE_CLASSMEM) + cxld->target_type = CXL_DECODER_HOSTONLYMEM; + else + cxld->target_type = CXL_DECODER_DEVMEM; + } else { + /* To be overridden by region type at commit time */ + cxld->target_type = CXL_DECODER_HOSTONLYMEM; + } + + if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) && + cxld->target_type == CXL_DECODER_HOSTONLYMEM) { + ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY; writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); } - cxld->target_type = CXL_DECODER_EXPANDER; } rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl), &cxld->interleave_ways); @@ -880,7 +899,7 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end, cxld->interleave_ways, cxld->interleave_granularity); - if (!info) { + if (!cxled) { lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which)); hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which)); target_list.value = (hi << 32) + lo; @@ -903,7 +922,6 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which)); hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which)); skip = (hi << 32) + lo; - cxled = to_cxl_endpoint_decoder(&cxld->dev); rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip); if (rc) { dev_err(&port->dev, diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index bea9cf31a12d..d6d067fbee97 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -182,7 +182,7 @@ static const char *cxl_mem_opcode_to_name(u16 opcode) /** * cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command - * @cxlds: The device data for the operation + * @mds: The driver data for the operation * @mbox_cmd: initialized command to execute * * Context: Any context. @@ -198,19 +198,19 @@ static const char *cxl_mem_opcode_to_name(u16 opcode) * error. While this distinction can be useful for commands from userspace, the * kernel will only be able to use results when both are successful. */ -int cxl_internal_send_cmd(struct cxl_dev_state *cxlds, +int cxl_internal_send_cmd(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *mbox_cmd) { size_t out_size, min_out; int rc; - if (mbox_cmd->size_in > cxlds->payload_size || - mbox_cmd->size_out > cxlds->payload_size) + if (mbox_cmd->size_in > mds->payload_size || + mbox_cmd->size_out > mds->payload_size) return -E2BIG; out_size = mbox_cmd->size_out; min_out = mbox_cmd->min_out; - rc = cxlds->mbox_send(cxlds, mbox_cmd); + rc = mds->mbox_send(mds, mbox_cmd); /* * EIO is reserved for a payload size mismatch and mbox_send() * may not return this error. @@ -220,7 +220,8 @@ int cxl_internal_send_cmd(struct cxl_dev_state *cxlds, if (rc) return rc; - if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) + if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS && + mbox_cmd->return_code != CXL_MBOX_CMD_RC_BACKGROUND) return cxl_mbox_cmd_rc2errno(mbox_cmd); if (!out_size) @@ -297,7 +298,7 @@ static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in) } static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox, - struct cxl_dev_state *cxlds, u16 opcode, + struct cxl_memdev_state *mds, u16 opcode, size_t in_size, size_t out_size, u64 in_payload) { *mbox = (struct cxl_mbox_cmd) { @@ -312,7 +313,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox, return PTR_ERR(mbox->payload_in); if (!cxl_payload_from_user_allowed(opcode, mbox->payload_in)) { - dev_dbg(cxlds->dev, "%s: input payload not allowed\n", + dev_dbg(mds->cxlds.dev, "%s: input payload not allowed\n", cxl_mem_opcode_to_name(opcode)); kvfree(mbox->payload_in); return -EBUSY; @@ -321,7 +322,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox, /* Prepare to handle a full payload for variable sized output */ if (out_size == CXL_VARIABLE_PAYLOAD) - mbox->size_out = cxlds->payload_size; + mbox->size_out = mds->payload_size; else mbox->size_out = out_size; @@ -343,7 +344,7 @@ static void cxl_mbox_cmd_dtor(struct cxl_mbox_cmd *mbox) static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd, const struct cxl_send_command *send_cmd, - struct cxl_dev_state *cxlds) + struct cxl_memdev_state *mds) { if (send_cmd->raw.rsvd) return -EINVAL; @@ -353,13 +354,13 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd, * gets passed along without further checking, so it must be * validated here. */ - if (send_cmd->out.size > cxlds->payload_size) + if (send_cmd->out.size > mds->payload_size) return -EINVAL; if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode)) return -EPERM; - dev_WARN_ONCE(cxlds->dev, true, "raw command path used\n"); + dev_WARN_ONCE(mds->cxlds.dev, true, "raw command path used\n"); *mem_cmd = (struct cxl_mem_command) { .info = { @@ -375,7 +376,7 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd, static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd, const struct cxl_send_command *send_cmd, - struct cxl_dev_state *cxlds) + struct cxl_memdev_state *mds) { struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id]; const struct cxl_command_info *info = &c->info; @@ -390,11 +391,11 @@ static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd, return -EINVAL; /* Check that the command is enabled for hardware */ - if (!test_bit(info->id, cxlds->enabled_cmds)) + if (!test_bit(info->id, mds->enabled_cmds)) return -ENOTTY; /* Check that the command is not claimed for exclusive kernel use */ - if (test_bit(info->id, cxlds->exclusive_cmds)) + if (test_bit(info->id, mds->exclusive_cmds)) return -EBUSY; /* Check the input buffer is the expected size */ @@ -423,7 +424,7 @@ static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd, /** * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND. * @mbox_cmd: Sanitized and populated &struct cxl_mbox_cmd. - * @cxlds: The device data for the operation + * @mds: The driver data for the operation * @send_cmd: &struct cxl_send_command copied in from userspace. * * Return: @@ -438,7 +439,7 @@ static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd, * safe to send to the hardware. */ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd, - struct cxl_dev_state *cxlds, + struct cxl_memdev_state *mds, const struct cxl_send_command *send_cmd) { struct cxl_mem_command mem_cmd; @@ -452,20 +453,20 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd, * supports, but output can be arbitrarily large (simply write out as * much data as the hardware provides). */ - if (send_cmd->in.size > cxlds->payload_size) + if (send_cmd->in.size > mds->payload_size) return -EINVAL; /* Sanitize and construct a cxl_mem_command */ if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) - rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, cxlds); + rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, mds); else - rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, cxlds); + rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, mds); if (rc) return rc; /* Sanitize and construct a cxl_mbox_cmd */ - return cxl_mbox_cmd_ctor(mbox_cmd, cxlds, mem_cmd.opcode, + return cxl_mbox_cmd_ctor(mbox_cmd, mds, mem_cmd.opcode, mem_cmd.info.size_in, mem_cmd.info.size_out, send_cmd->in.payload); } @@ -473,6 +474,7 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd, int cxl_query_cmd(struct cxl_memdev *cxlmd, struct cxl_mem_query_commands __user *q) { + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); struct device *dev = &cxlmd->dev; struct cxl_mem_command *cmd; u32 n_commands; @@ -494,9 +496,9 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd, cxl_for_each_cmd(cmd) { struct cxl_command_info info = cmd->info; - if (test_bit(info.id, cxlmd->cxlds->enabled_cmds)) + if (test_bit(info.id, mds->enabled_cmds)) info.flags |= CXL_MEM_COMMAND_FLAG_ENABLED; - if (test_bit(info.id, cxlmd->cxlds->exclusive_cmds)) + if (test_bit(info.id, mds->exclusive_cmds)) info.flags |= CXL_MEM_COMMAND_FLAG_EXCLUSIVE; if (copy_to_user(&q->commands[j++], &info, sizeof(info))) @@ -511,7 +513,7 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd, /** * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace. - * @cxlds: The device data for the operation + * @mds: The driver data for the operation * @mbox_cmd: The validated mailbox command. * @out_payload: Pointer to userspace's output payload. * @size_out: (Input) Max payload size to copy out. @@ -532,12 +534,12 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd, * * See cxl_send_cmd(). */ -static int handle_mailbox_cmd_from_user(struct cxl_dev_state *cxlds, +static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *mbox_cmd, u64 out_payload, s32 *size_out, u32 *retval) { - struct device *dev = cxlds->dev; + struct device *dev = mds->cxlds.dev; int rc; dev_dbg(dev, @@ -547,7 +549,7 @@ static int handle_mailbox_cmd_from_user(struct cxl_dev_state *cxlds, cxl_mem_opcode_to_name(mbox_cmd->opcode), mbox_cmd->opcode, mbox_cmd->size_in); - rc = cxlds->mbox_send(cxlds, mbox_cmd); + rc = mds->mbox_send(mds, mbox_cmd); if (rc) goto out; @@ -576,7 +578,7 @@ out: int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s) { - struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); struct device *dev = &cxlmd->dev; struct cxl_send_command send; struct cxl_mbox_cmd mbox_cmd; @@ -587,11 +589,11 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s) if (copy_from_user(&send, s, sizeof(send))) return -EFAULT; - rc = cxl_validate_cmd_from_user(&mbox_cmd, cxlmd->cxlds, &send); + rc = cxl_validate_cmd_from_user(&mbox_cmd, mds, &send); if (rc) return rc; - rc = handle_mailbox_cmd_from_user(cxlds, &mbox_cmd, send.out.payload, + rc = handle_mailbox_cmd_from_user(mds, &mbox_cmd, send.out.payload, &send.out.size, &send.retval); if (rc) return rc; @@ -602,13 +604,14 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s) return 0; } -static int cxl_xfer_log(struct cxl_dev_state *cxlds, uuid_t *uuid, u32 *size, u8 *out) +static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid, + u32 *size, u8 *out) { u32 remaining = *size; u32 offset = 0; while (remaining) { - u32 xfer_size = min_t(u32, remaining, cxlds->payload_size); + u32 xfer_size = min_t(u32, remaining, mds->payload_size); struct cxl_mbox_cmd mbox_cmd; struct cxl_mbox_get_log log; int rc; @@ -627,7 +630,7 @@ static int cxl_xfer_log(struct cxl_dev_state *cxlds, uuid_t *uuid, u32 *size, u8 .payload_out = out, }; - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); /* * The output payload length that indicates the number @@ -654,17 +657,18 @@ static int cxl_xfer_log(struct cxl_dev_state *cxlds, uuid_t *uuid, u32 *size, u8 /** * cxl_walk_cel() - Walk through the Command Effects Log. - * @cxlds: The device data for the operation + * @mds: The driver data for the operation * @size: Length of the Command Effects Log. * @cel: CEL * * Iterate over each entry in the CEL and determine if the driver supports the * command. If so, the command is enabled for the device and can be used later. */ -static void cxl_walk_cel(struct cxl_dev_state *cxlds, size_t size, u8 *cel) +static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel) { struct cxl_cel_entry *cel_entry; const int cel_entries = size / sizeof(*cel_entry); + struct device *dev = mds->cxlds.dev; int i; cel_entry = (struct cxl_cel_entry *) cel; @@ -674,39 +678,39 @@ static void cxl_walk_cel(struct cxl_dev_state *cxlds, size_t size, u8 *cel) struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); if (!cmd && !cxl_is_poison_command(opcode)) { - dev_dbg(cxlds->dev, + dev_dbg(dev, "Opcode 0x%04x unsupported by driver\n", opcode); continue; } if (cmd) - set_bit(cmd->info.id, cxlds->enabled_cmds); + set_bit(cmd->info.id, mds->enabled_cmds); if (cxl_is_poison_command(opcode)) - cxl_set_poison_cmd_enabled(&cxlds->poison, opcode); + cxl_set_poison_cmd_enabled(&mds->poison, opcode); - dev_dbg(cxlds->dev, "Opcode 0x%04x enabled\n", opcode); + dev_dbg(dev, "Opcode 0x%04x enabled\n", opcode); } } -static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_dev_state *cxlds) +static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds) { struct cxl_mbox_get_supported_logs *ret; struct cxl_mbox_cmd mbox_cmd; int rc; - ret = kvmalloc(cxlds->payload_size, GFP_KERNEL); + ret = kvmalloc(mds->payload_size, GFP_KERNEL); if (!ret) return ERR_PTR(-ENOMEM); mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS, - .size_out = cxlds->payload_size, + .size_out = mds->payload_size, .payload_out = ret, /* At least the record number field must be valid */ .min_out = 2, }; - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); if (rc < 0) { kvfree(ret); return ERR_PTR(rc); @@ -729,22 +733,22 @@ static const uuid_t log_uuid[] = { /** * cxl_enumerate_cmds() - Enumerate commands for a device. - * @cxlds: The device data for the operation + * @mds: The driver data for the operation * * Returns 0 if enumerate completed successfully. * * CXL devices have optional support for certain commands. This function will * determine the set of supported commands for the hardware and update the - * enabled_cmds bitmap in the @cxlds. + * enabled_cmds bitmap in the @mds. */ -int cxl_enumerate_cmds(struct cxl_dev_state *cxlds) +int cxl_enumerate_cmds(struct cxl_memdev_state *mds) { struct cxl_mbox_get_supported_logs *gsl; - struct device *dev = cxlds->dev; + struct device *dev = mds->cxlds.dev; struct cxl_mem_command *cmd; int i, rc; - gsl = cxl_get_gsl(cxlds); + gsl = cxl_get_gsl(mds); if (IS_ERR(gsl)) return PTR_ERR(gsl); @@ -765,19 +769,19 @@ int cxl_enumerate_cmds(struct cxl_dev_state *cxlds) goto out; } - rc = cxl_xfer_log(cxlds, &uuid, &size, log); + rc = cxl_xfer_log(mds, &uuid, &size, log); if (rc) { kvfree(log); goto out; } - cxl_walk_cel(cxlds, size, log); + cxl_walk_cel(mds, size, log); kvfree(log); /* In case CEL was bogus, enable some default commands. */ cxl_for_each_cmd(cmd) if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE) - set_bit(cmd->info.id, cxlds->enabled_cmds); + set_bit(cmd->info.id, mds->enabled_cmds); /* Found the required CEL */ rc = 0; @@ -838,7 +842,7 @@ static void cxl_event_trace_record(const struct cxl_memdev *cxlmd, } } -static int cxl_clear_event_record(struct cxl_dev_state *cxlds, +static int cxl_clear_event_record(struct cxl_memdev_state *mds, enum cxl_event_log_type log, struct cxl_get_event_payload *get_pl) { @@ -852,9 +856,9 @@ static int cxl_clear_event_record(struct cxl_dev_state *cxlds, int i; /* Payload size may limit the max handles */ - if (pl_size > cxlds->payload_size) { - max_handles = (cxlds->payload_size - sizeof(*payload)) / - sizeof(__le16); + if (pl_size > mds->payload_size) { + max_handles = (mds->payload_size - sizeof(*payload)) / + sizeof(__le16); pl_size = struct_size(payload, handles, max_handles); } @@ -879,12 +883,12 @@ static int cxl_clear_event_record(struct cxl_dev_state *cxlds, i = 0; for (cnt = 0; cnt < total; cnt++) { payload->handles[i++] = get_pl->records[cnt].hdr.handle; - dev_dbg(cxlds->dev, "Event log '%d': Clearing %u\n", - log, le16_to_cpu(payload->handles[i])); + dev_dbg(mds->cxlds.dev, "Event log '%d': Clearing %u\n", log, + le16_to_cpu(payload->handles[i])); if (i == max_handles) { payload->nr_recs = i; - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); if (rc) goto free_pl; i = 0; @@ -895,7 +899,7 @@ static int cxl_clear_event_record(struct cxl_dev_state *cxlds, if (i) { payload->nr_recs = i; mbox_cmd.size_in = struct_size(payload, handles, i); - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); if (rc) goto free_pl; } @@ -905,32 +909,34 @@ free_pl: return rc; } -static void cxl_mem_get_records_log(struct cxl_dev_state *cxlds, +static void cxl_mem_get_records_log(struct cxl_memdev_state *mds, enum cxl_event_log_type type) { + struct cxl_memdev *cxlmd = mds->cxlds.cxlmd; + struct device *dev = mds->cxlds.dev; struct cxl_get_event_payload *payload; struct cxl_mbox_cmd mbox_cmd; u8 log_type = type; u16 nr_rec; - mutex_lock(&cxlds->event.log_lock); - payload = cxlds->event.buf; + mutex_lock(&mds->event.log_lock); + payload = mds->event.buf; mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_GET_EVENT_RECORD, .payload_in = &log_type, .size_in = sizeof(log_type), .payload_out = payload, - .size_out = cxlds->payload_size, + .size_out = mds->payload_size, .min_out = struct_size(payload, records, 0), }; do { int rc, i; - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); if (rc) { - dev_err_ratelimited(cxlds->dev, + dev_err_ratelimited(dev, "Event log '%d': Failed to query event records : %d", type, rc); break; @@ -941,27 +947,27 @@ static void cxl_mem_get_records_log(struct cxl_dev_state *cxlds, break; for (i = 0; i < nr_rec; i++) - cxl_event_trace_record(cxlds->cxlmd, type, + cxl_event_trace_record(cxlmd, type, &payload->records[i]); if (payload->flags & CXL_GET_EVENT_FLAG_OVERFLOW) - trace_cxl_overflow(cxlds->cxlmd, type, payload); + trace_cxl_overflow(cxlmd, type, payload); - rc = cxl_clear_event_record(cxlds, type, payload); + rc = cxl_clear_event_record(mds, type, payload); if (rc) { - dev_err_ratelimited(cxlds->dev, + dev_err_ratelimited(dev, "Event log '%d': Failed to clear events : %d", type, rc); break; } } while (nr_rec); - mutex_unlock(&cxlds->event.log_lock); + mutex_unlock(&mds->event.log_lock); } /** * cxl_mem_get_event_records - Get Event Records from the device - * @cxlds: The device data for the operation + * @mds: The driver data for the operation * @status: Event Status register value identifying which events are available. * * Retrieve all event records available on the device, report them as trace @@ -970,24 +976,24 @@ static void cxl_mem_get_records_log(struct cxl_dev_state *cxlds, * See CXL rev 3.0 @8.2.9.2.2 Get Event Records * See CXL rev 3.0 @8.2.9.2.3 Clear Event Records */ -void cxl_mem_get_event_records(struct cxl_dev_state *cxlds, u32 status) +void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status) { - dev_dbg(cxlds->dev, "Reading event logs: %x\n", status); + dev_dbg(mds->cxlds.dev, "Reading event logs: %x\n", status); if (status & CXLDEV_EVENT_STATUS_FATAL) - cxl_mem_get_records_log(cxlds, CXL_EVENT_TYPE_FATAL); + cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FATAL); if (status & CXLDEV_EVENT_STATUS_FAIL) - cxl_mem_get_records_log(cxlds, CXL_EVENT_TYPE_FAIL); + cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FAIL); if (status & CXLDEV_EVENT_STATUS_WARN) - cxl_mem_get_records_log(cxlds, CXL_EVENT_TYPE_WARN); + cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_WARN); if (status & CXLDEV_EVENT_STATUS_INFO) - cxl_mem_get_records_log(cxlds, CXL_EVENT_TYPE_INFO); + cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_INFO); } EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, CXL); /** * cxl_mem_get_partition_info - Get partition info - * @cxlds: The device data for the operation + * @mds: The driver data for the operation * * Retrieve the current partition info for the device specified. The active * values are the current capacity in bytes. If not 0, the 'next' values are @@ -997,7 +1003,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, CXL); * * See CXL @8.2.9.5.2.1 Get Partition Info */ -static int cxl_mem_get_partition_info(struct cxl_dev_state *cxlds) +static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds) { struct cxl_mbox_get_partition_info pi; struct cxl_mbox_cmd mbox_cmd; @@ -1008,17 +1014,17 @@ static int cxl_mem_get_partition_info(struct cxl_dev_state *cxlds) .size_out = sizeof(pi), .payload_out = &pi, }; - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); if (rc) return rc; - cxlds->active_volatile_bytes = + mds->active_volatile_bytes = le64_to_cpu(pi.active_volatile_cap) * CXL_CAPACITY_MULTIPLIER; - cxlds->active_persistent_bytes = + mds->active_persistent_bytes = le64_to_cpu(pi.active_persistent_cap) * CXL_CAPACITY_MULTIPLIER; - cxlds->next_volatile_bytes = + mds->next_volatile_bytes = le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER; - cxlds->next_persistent_bytes = + mds->next_persistent_bytes = le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER; return 0; @@ -1026,14 +1032,14 @@ static int cxl_mem_get_partition_info(struct cxl_dev_state *cxlds) /** * cxl_dev_state_identify() - Send the IDENTIFY command to the device. - * @cxlds: The device data for the operation + * @mds: The driver data for the operation * * Return: 0 if identify was executed successfully or media not ready. * * This will dispatch the identify command to the device and on success populate * structures to be exported to sysfs. */ -int cxl_dev_state_identify(struct cxl_dev_state *cxlds) +int cxl_dev_state_identify(struct cxl_memdev_state *mds) { /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */ struct cxl_mbox_identify id; @@ -1041,7 +1047,7 @@ int cxl_dev_state_identify(struct cxl_dev_state *cxlds) u32 val; int rc; - if (!cxlds->media_ready) + if (!mds->cxlds.media_ready) return 0; mbox_cmd = (struct cxl_mbox_cmd) { @@ -1049,31 +1055,92 @@ int cxl_dev_state_identify(struct cxl_dev_state *cxlds) .size_out = sizeof(id), .payload_out = &id, }; - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); if (rc < 0) return rc; - cxlds->total_bytes = + mds->total_bytes = le64_to_cpu(id.total_capacity) * CXL_CAPACITY_MULTIPLIER; - cxlds->volatile_only_bytes = + mds->volatile_only_bytes = le64_to_cpu(id.volatile_capacity) * CXL_CAPACITY_MULTIPLIER; - cxlds->persistent_only_bytes = + mds->persistent_only_bytes = le64_to_cpu(id.persistent_capacity) * CXL_CAPACITY_MULTIPLIER; - cxlds->partition_align_bytes = + mds->partition_align_bytes = le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER; - cxlds->lsa_size = le32_to_cpu(id.lsa_size); - memcpy(cxlds->firmware_version, id.fw_revision, sizeof(id.fw_revision)); + mds->lsa_size = le32_to_cpu(id.lsa_size); + memcpy(mds->firmware_version, id.fw_revision, + sizeof(id.fw_revision)); - if (test_bit(CXL_POISON_ENABLED_LIST, cxlds->poison.enabled_cmds)) { + if (test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) { val = get_unaligned_le24(id.poison_list_max_mer); - cxlds->poison.max_errors = min_t(u32, val, CXL_POISON_LIST_MAX); + mds->poison.max_errors = min_t(u32, val, CXL_POISON_LIST_MAX); } return 0; } EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL); +/** + * cxl_mem_sanitize() - Send a sanitization command to the device. + * @mds: The device data for the operation + * @cmd: The specific sanitization command opcode + * + * Return: 0 if the command was executed successfully, regardless of + * whether or not the actual security operation is done in the background, + * such as for the Sanitize case. + * Error return values can be the result of the mailbox command, -EINVAL + * when security requirements are not met or invalid contexts. + * + * See CXL 3.0 @8.2.9.8.5.1 Sanitize and @8.2.9.8.5.2 Secure Erase. + */ +int cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd) +{ + int rc; + u32 sec_out = 0; + struct cxl_get_security_output { + __le32 flags; + } out; + struct cxl_mbox_cmd sec_cmd = { + .opcode = CXL_MBOX_OP_GET_SECURITY_STATE, + .payload_out = &out, + .size_out = sizeof(out), + }; + struct cxl_mbox_cmd mbox_cmd = { .opcode = cmd }; + struct cxl_dev_state *cxlds = &mds->cxlds; + + if (cmd != CXL_MBOX_OP_SANITIZE && cmd != CXL_MBOX_OP_SECURE_ERASE) + return -EINVAL; + + rc = cxl_internal_send_cmd(mds, &sec_cmd); + if (rc < 0) { + dev_err(cxlds->dev, "Failed to get security state : %d", rc); + return rc; + } + + /* + * Prior to using these commands, any security applied to + * the user data areas of the device shall be DISABLED (or + * UNLOCKED for secure erase case). + */ + sec_out = le32_to_cpu(out.flags); + if (sec_out & CXL_PMEM_SEC_STATE_USER_PASS_SET) + return -EINVAL; + + if (cmd == CXL_MBOX_OP_SECURE_ERASE && + sec_out & CXL_PMEM_SEC_STATE_LOCKED) + return -EINVAL; + + rc = cxl_internal_send_cmd(mds, &mbox_cmd); + if (rc < 0) { + dev_err(cxlds->dev, "Failed to sanitize device : %d", rc); + return rc; + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_mem_sanitize, CXL); + static int add_dpa_res(struct device *dev, struct resource *parent, struct resource *res, resource_size_t start, resource_size_t size, const char *type) @@ -1100,8 +1167,9 @@ static int add_dpa_res(struct device *dev, struct resource *parent, return 0; } -int cxl_mem_create_range_info(struct cxl_dev_state *cxlds) +int cxl_mem_create_range_info(struct cxl_memdev_state *mds) { + struct cxl_dev_state *cxlds = &mds->cxlds; struct device *dev = cxlds->dev; int rc; @@ -1113,35 +1181,35 @@ int cxl_mem_create_range_info(struct cxl_dev_state *cxlds) } cxlds->dpa_res = - (struct resource)DEFINE_RES_MEM(0, cxlds->total_bytes); + (struct resource)DEFINE_RES_MEM(0, mds->total_bytes); - if (cxlds->partition_align_bytes == 0) { + if (mds->partition_align_bytes == 0) { rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0, - cxlds->volatile_only_bytes, "ram"); + mds->volatile_only_bytes, "ram"); if (rc) return rc; return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res, - cxlds->volatile_only_bytes, - cxlds->persistent_only_bytes, "pmem"); + mds->volatile_only_bytes, + mds->persistent_only_bytes, "pmem"); } - rc = cxl_mem_get_partition_info(cxlds); + rc = cxl_mem_get_partition_info(mds); if (rc) { dev_err(dev, "Failed to query partition information\n"); return rc; } rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0, - cxlds->active_volatile_bytes, "ram"); + mds->active_volatile_bytes, "ram"); if (rc) return rc; return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res, - cxlds->active_volatile_bytes, - cxlds->active_persistent_bytes, "pmem"); + mds->active_volatile_bytes, + mds->active_persistent_bytes, "pmem"); } EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL); -int cxl_set_timestamp(struct cxl_dev_state *cxlds) +int cxl_set_timestamp(struct cxl_memdev_state *mds) { struct cxl_mbox_cmd mbox_cmd; struct cxl_mbox_set_timestamp_in pi; @@ -1154,7 +1222,7 @@ int cxl_set_timestamp(struct cxl_dev_state *cxlds) .payload_in = &pi, }; - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); /* * Command is optional. Devices may have another way of providing * a timestamp, or may return all 0s in timestamp fields. @@ -1170,18 +1238,18 @@ EXPORT_SYMBOL_NS_GPL(cxl_set_timestamp, CXL); int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, struct cxl_region *cxlr) { - struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); struct cxl_mbox_poison_out *po; struct cxl_mbox_poison_in pi; struct cxl_mbox_cmd mbox_cmd; int nr_records = 0; int rc; - rc = mutex_lock_interruptible(&cxlds->poison.lock); + rc = mutex_lock_interruptible(&mds->poison.lock); if (rc) return rc; - po = cxlds->poison.list_out; + po = mds->poison.list_out; pi.offset = cpu_to_le64(offset); pi.length = cpu_to_le64(len / CXL_POISON_LEN_MULT); @@ -1189,13 +1257,13 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, .opcode = CXL_MBOX_OP_GET_POISON, .size_in = sizeof(pi), .payload_in = &pi, - .size_out = cxlds->payload_size, + .size_out = mds->payload_size, .payload_out = po, .min_out = struct_size(po, record, 0), }; do { - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); if (rc) break; @@ -1206,14 +1274,14 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, /* Protect against an uncleared _FLAG_MORE */ nr_records = nr_records + le16_to_cpu(po->count); - if (nr_records >= cxlds->poison.max_errors) { + if (nr_records >= mds->poison.max_errors) { dev_dbg(&cxlmd->dev, "Max Error Records reached: %d\n", nr_records); break; } } while (po->flags & CXL_POISON_FLAG_MORE); - mutex_unlock(&cxlds->poison.lock); + mutex_unlock(&mds->poison.lock); return rc; } EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, CXL); @@ -1223,52 +1291,53 @@ static void free_poison_buf(void *buf) kvfree(buf); } -/* Get Poison List output buffer is protected by cxlds->poison.lock */ -static int cxl_poison_alloc_buf(struct cxl_dev_state *cxlds) +/* Get Poison List output buffer is protected by mds->poison.lock */ +static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds) { - cxlds->poison.list_out = kvmalloc(cxlds->payload_size, GFP_KERNEL); - if (!cxlds->poison.list_out) + mds->poison.list_out = kvmalloc(mds->payload_size, GFP_KERNEL); + if (!mds->poison.list_out) return -ENOMEM; - return devm_add_action_or_reset(cxlds->dev, free_poison_buf, - cxlds->poison.list_out); + return devm_add_action_or_reset(mds->cxlds.dev, free_poison_buf, + mds->poison.list_out); } -int cxl_poison_state_init(struct cxl_dev_state *cxlds) +int cxl_poison_state_init(struct cxl_memdev_state *mds) { int rc; - if (!test_bit(CXL_POISON_ENABLED_LIST, cxlds->poison.enabled_cmds)) + if (!test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) return 0; - rc = cxl_poison_alloc_buf(cxlds); + rc = cxl_poison_alloc_buf(mds); if (rc) { - clear_bit(CXL_POISON_ENABLED_LIST, cxlds->poison.enabled_cmds); + clear_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds); return rc; } - mutex_init(&cxlds->poison.lock); + mutex_init(&mds->poison.lock); return 0; } EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, CXL); -struct cxl_dev_state *cxl_dev_state_create(struct device *dev) +struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev) { - struct cxl_dev_state *cxlds; + struct cxl_memdev_state *mds; - cxlds = devm_kzalloc(dev, sizeof(*cxlds), GFP_KERNEL); - if (!cxlds) { + mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL); + if (!mds) { dev_err(dev, "No memory available\n"); return ERR_PTR(-ENOMEM); } - mutex_init(&cxlds->mbox_mutex); - mutex_init(&cxlds->event.log_lock); - cxlds->dev = dev; + mutex_init(&mds->mbox_mutex); + mutex_init(&mds->event.log_lock); + mds->cxlds.dev = dev; + mds->cxlds.type = CXL_DEVTYPE_CLASSMEM; - return cxlds; + return mds; } -EXPORT_SYMBOL_NS_GPL(cxl_dev_state_create, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, CXL); void __init cxl_mbox_init(void) { diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index 057a43267290..f99e7ec3cc40 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2020 Intel Corporation. */ +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/firmware.h> #include <linux/device.h> #include <linux/slab.h> #include <linux/idr.h> @@ -39,8 +41,11 @@ static ssize_t firmware_version_show(struct device *dev, { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - return sysfs_emit(buf, "%.16s\n", cxlds->firmware_version); + if (!mds) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%.16s\n", mds->firmware_version); } static DEVICE_ATTR_RO(firmware_version); @@ -49,8 +54,11 @@ static ssize_t payload_max_show(struct device *dev, { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - return sysfs_emit(buf, "%zu\n", cxlds->payload_size); + if (!mds) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%zu\n", mds->payload_size); } static DEVICE_ATTR_RO(payload_max); @@ -59,8 +67,11 @@ static ssize_t label_storage_size_show(struct device *dev, { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - return sysfs_emit(buf, "%zu\n", cxlds->lsa_size); + if (!mds) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%zu\n", mds->lsa_size); } static DEVICE_ATTR_RO(label_storage_size); @@ -107,6 +118,89 @@ static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RO(numa_node); +static ssize_t security_state_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + u64 reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET); + u32 pct = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK, reg); + u16 cmd = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg); + unsigned long state = mds->security.state; + + if (cmd == CXL_MBOX_OP_SANITIZE && pct != 100) + return sysfs_emit(buf, "sanitize\n"); + + if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) + return sysfs_emit(buf, "disabled\n"); + if (state & CXL_PMEM_SEC_STATE_FROZEN || + state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || + state & CXL_PMEM_SEC_STATE_USER_PLIMIT) + return sysfs_emit(buf, "frozen\n"); + if (state & CXL_PMEM_SEC_STATE_LOCKED) + return sysfs_emit(buf, "locked\n"); + else + return sysfs_emit(buf, "unlocked\n"); +} +static struct device_attribute dev_attr_security_state = + __ATTR(state, 0444, security_state_show, NULL); + +static ssize_t security_sanitize_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + struct cxl_port *port = cxlmd->endpoint; + bool sanitize; + ssize_t rc; + + if (kstrtobool(buf, &sanitize) || !sanitize) + return -EINVAL; + + if (!port || !is_cxl_endpoint(port)) + return -EINVAL; + + /* ensure no regions are mapped to this memdev */ + if (port->commit_end != -1) + return -EBUSY; + + rc = cxl_mem_sanitize(mds, CXL_MBOX_OP_SANITIZE); + + return rc ? rc : len; +} +static struct device_attribute dev_attr_security_sanitize = + __ATTR(sanitize, 0200, NULL, security_sanitize_store); + +static ssize_t security_erase_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + struct cxl_port *port = cxlmd->endpoint; + ssize_t rc; + bool erase; + + if (kstrtobool(buf, &erase) || !erase) + return -EINVAL; + + if (!port || !is_cxl_endpoint(port)) + return -EINVAL; + + /* ensure no regions are mapped to this memdev */ + if (port->commit_end != -1) + return -EBUSY; + + rc = cxl_mem_sanitize(mds, CXL_MBOX_OP_SECURE_ERASE); + + return rc ? rc : len; +} +static struct device_attribute dev_attr_security_erase = + __ATTR(erase, 0200, NULL, security_erase_store); + static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) { struct cxl_dev_state *cxlds = cxlmd->cxlds; @@ -140,7 +234,7 @@ int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) struct cxl_port *port; int rc; - port = dev_get_drvdata(&cxlmd->dev); + port = cxlmd->endpoint; if (!port || !is_cxl_endpoint(port)) return -EINVAL; @@ -198,7 +292,7 @@ static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa) ctx = (struct cxl_dpa_to_region_context) { .dpa = dpa, }; - port = dev_get_drvdata(&cxlmd->dev); + port = cxlmd->endpoint; if (port && is_cxl_endpoint(port) && port->commit_end != -1) device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region); @@ -231,7 +325,7 @@ static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) { - struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); struct cxl_mbox_inject_poison inject; struct cxl_poison_record record; struct cxl_mbox_cmd mbox_cmd; @@ -255,13 +349,13 @@ int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) .size_in = sizeof(inject), .payload_in = &inject, }; - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); if (rc) goto out; cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) - dev_warn_once(cxlds->dev, + dev_warn_once(mds->cxlds.dev, "poison inject dpa:%#llx region: %s\n", dpa, dev_name(&cxlr->dev)); @@ -279,7 +373,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL); int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) { - struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); struct cxl_mbox_clear_poison clear; struct cxl_poison_record record; struct cxl_mbox_cmd mbox_cmd; @@ -312,14 +406,15 @@ int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) .payload_in = &clear, }; - rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); + rc = cxl_internal_send_cmd(mds, &mbox_cmd); if (rc) goto out; cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) - dev_warn_once(cxlds->dev, "poison clear dpa:%#llx region: %s\n", - dpa, dev_name(&cxlr->dev)); + dev_warn_once(mds->cxlds.dev, + "poison clear dpa:%#llx region: %s\n", dpa, + dev_name(&cxlr->dev)); record = (struct cxl_poison_record) { .address = cpu_to_le64(dpa), @@ -352,6 +447,13 @@ static struct attribute *cxl_memdev_ram_attributes[] = { NULL, }; +static struct attribute *cxl_memdev_security_attributes[] = { + &dev_attr_security_state.attr, + &dev_attr_security_sanitize.attr, + &dev_attr_security_erase.attr, + NULL, +}; + static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, int n) { @@ -375,10 +477,16 @@ static struct attribute_group cxl_memdev_pmem_attribute_group = { .attrs = cxl_memdev_pmem_attributes, }; +static struct attribute_group cxl_memdev_security_attribute_group = { + .name = "security", + .attrs = cxl_memdev_security_attributes, +}; + static const struct attribute_group *cxl_memdev_attribute_groups[] = { &cxl_memdev_attribute_group, &cxl_memdev_ram_attribute_group, &cxl_memdev_pmem_attribute_group, + &cxl_memdev_security_attribute_group, NULL, }; @@ -397,17 +505,18 @@ EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL); /** * set_exclusive_cxl_commands() - atomically disable user cxl commands - * @cxlds: The device state to operate on + * @mds: The device state to operate on * @cmds: bitmap of commands to mark exclusive * * Grab the cxl_memdev_rwsem in write mode to flush in-flight * invocations of the ioctl path and then disable future execution of * commands with the command ids set in @cmds. */ -void set_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds) +void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, + unsigned long *cmds) { down_write(&cxl_memdev_rwsem); - bitmap_or(cxlds->exclusive_cmds, cxlds->exclusive_cmds, cmds, + bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds, CXL_MEM_COMMAND_ID_MAX); up_write(&cxl_memdev_rwsem); } @@ -415,23 +524,34 @@ EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL); /** * clear_exclusive_cxl_commands() - atomically enable user cxl commands - * @cxlds: The device state to modify + * @mds: The device state to modify * @cmds: bitmap of commands to mark available for userspace */ -void clear_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds) +void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, + unsigned long *cmds) { down_write(&cxl_memdev_rwsem); - bitmap_andnot(cxlds->exclusive_cmds, cxlds->exclusive_cmds, cmds, + bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds, CXL_MEM_COMMAND_ID_MAX); up_write(&cxl_memdev_rwsem); } EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL); +static void cxl_memdev_security_shutdown(struct device *dev) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + + if (mds->security.poll) + cancel_delayed_work_sync(&mds->security.poll_dwork); +} + static void cxl_memdev_shutdown(struct device *dev) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); down_write(&cxl_memdev_rwsem); + cxl_memdev_security_shutdown(dev); cxlmd->cxlds = NULL; up_write(&cxl_memdev_rwsem); } @@ -511,10 +631,12 @@ static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct cxl_memdev *cxlmd = file->private_data; + struct cxl_dev_state *cxlds; int rc = -ENXIO; down_read(&cxl_memdev_rwsem); - if (cxlmd->cxlds) + cxlds = cxlmd->cxlds; + if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); up_read(&cxl_memdev_rwsem); @@ -542,6 +664,316 @@ static int cxl_memdev_release_file(struct inode *inode, struct file *file) return 0; } +/** + * cxl_mem_get_fw_info - Get Firmware info + * @mds: The device data for the operation + * + * Retrieve firmware info for the device specified. + * + * Return: 0 if no error: or the result of the mailbox command. + * + * See CXL-3.0 8.2.9.3.1 Get FW Info + */ +static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) +{ + struct cxl_mbox_get_fw_info info; + struct cxl_mbox_cmd mbox_cmd; + int rc; + + mbox_cmd = (struct cxl_mbox_cmd) { + .opcode = CXL_MBOX_OP_GET_FW_INFO, + .size_out = sizeof(info), + .payload_out = &info, + }; + + rc = cxl_internal_send_cmd(mds, &mbox_cmd); + if (rc < 0) + return rc; + + mds->fw.num_slots = info.num_slots; + mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, + info.slot_info); + + return 0; +} + +/** + * cxl_mem_activate_fw - Activate Firmware + * @mds: The device data for the operation + * @slot: slot number to activate + * + * Activate firmware in a given slot for the device specified. + * + * Return: 0 if no error: or the result of the mailbox command. + * + * See CXL-3.0 8.2.9.3.3 Activate FW + */ +static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) +{ + struct cxl_mbox_activate_fw activate; + struct cxl_mbox_cmd mbox_cmd; + + if (slot == 0 || slot > mds->fw.num_slots) + return -EINVAL; + + mbox_cmd = (struct cxl_mbox_cmd) { + .opcode = CXL_MBOX_OP_ACTIVATE_FW, + .size_in = sizeof(activate), + .payload_in = &activate, + }; + + /* Only offline activation supported for now */ + activate.action = CXL_FW_ACTIVATE_OFFLINE; + activate.slot = slot; + + return cxl_internal_send_cmd(mds, &mbox_cmd); +} + +/** + * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer + * @mds: The device data for the operation + * + * Abort an in-progress firmware transfer for the device specified. + * + * Return: 0 if no error: or the result of the mailbox command. + * + * See CXL-3.0 8.2.9.3.2 Transfer FW + */ +static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) +{ + struct cxl_mbox_transfer_fw *transfer; + struct cxl_mbox_cmd mbox_cmd; + int rc; + + transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); + if (!transfer) + return -ENOMEM; + + /* Set a 1s poll interval and a total wait time of 30s */ + mbox_cmd = (struct cxl_mbox_cmd) { + .opcode = CXL_MBOX_OP_TRANSFER_FW, + .size_in = sizeof(*transfer), + .payload_in = transfer, + .poll_interval_ms = 1000, + .poll_count = 30, + }; + + transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; + + rc = cxl_internal_send_cmd(mds, &mbox_cmd); + kfree(transfer); + return rc; +} + +static void cxl_fw_cleanup(struct fw_upload *fwl) +{ + struct cxl_memdev_state *mds = fwl->dd_handle; + + mds->fw.next_slot = 0; +} + +static int cxl_fw_do_cancel(struct fw_upload *fwl) +{ + struct cxl_memdev_state *mds = fwl->dd_handle; + struct cxl_dev_state *cxlds = &mds->cxlds; + struct cxl_memdev *cxlmd = cxlds->cxlmd; + int rc; + + rc = cxl_mem_abort_fw_xfer(mds); + if (rc < 0) + dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); + + return FW_UPLOAD_ERR_CANCELED; +} + +static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, + u32 size) +{ + struct cxl_memdev_state *mds = fwl->dd_handle; + struct cxl_mbox_transfer_fw *transfer; + + if (!size) + return FW_UPLOAD_ERR_INVALID_SIZE; + + mds->fw.oneshot = struct_size(transfer, data, size) < + mds->payload_size; + + if (cxl_mem_get_fw_info(mds)) + return FW_UPLOAD_ERR_HW_ERROR; + + /* + * So far no state has been changed, hence no other cleanup is + * necessary. Simply return the cancelled status. + */ + if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) + return FW_UPLOAD_ERR_CANCELED; + + return FW_UPLOAD_ERR_NONE; +} + +static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, + u32 offset, u32 size, u32 *written) +{ + struct cxl_memdev_state *mds = fwl->dd_handle; + struct cxl_dev_state *cxlds = &mds->cxlds; + struct cxl_memdev *cxlmd = cxlds->cxlmd; + struct cxl_mbox_transfer_fw *transfer; + struct cxl_mbox_cmd mbox_cmd; + u32 cur_size, remaining; + size_t size_in; + int rc; + + *written = 0; + + /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ + if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { + dev_err(&cxlmd->dev, + "misaligned offset for FW transfer slice (%u)\n", + offset); + return FW_UPLOAD_ERR_RW_ERROR; + } + + /* + * Pick transfer size based on mds->payload_size @size must bw 128-byte + * aligned, ->payload_size is a power of 2 starting at 256 bytes, and + * sizeof(*transfer) is 128. These constraints imply that @cur_size + * will always be 128b aligned. + */ + cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer)); + + remaining = size - cur_size; + size_in = struct_size(transfer, data, cur_size); + + if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) + return cxl_fw_do_cancel(fwl); + + /* + * Slot numbers are 1-indexed + * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') + * Check for rollover using modulo, and 1-index it by adding 1 + */ + mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1; + + /* Do the transfer via mailbox cmd */ + transfer = kzalloc(size_in, GFP_KERNEL); + if (!transfer) + return FW_UPLOAD_ERR_RW_ERROR; + + transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); + memcpy(transfer->data, data + offset, cur_size); + if (mds->fw.oneshot) { + transfer->action = CXL_FW_TRANSFER_ACTION_FULL; + transfer->slot = mds->fw.next_slot; + } else { + if (offset == 0) { + transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; + } else if (remaining == 0) { + transfer->action = CXL_FW_TRANSFER_ACTION_END; + transfer->slot = mds->fw.next_slot; + } else { + transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; + } + } + + mbox_cmd = (struct cxl_mbox_cmd) { + .opcode = CXL_MBOX_OP_TRANSFER_FW, + .size_in = size_in, + .payload_in = transfer, + .poll_interval_ms = 1000, + .poll_count = 30, + }; + + rc = cxl_internal_send_cmd(mds, &mbox_cmd); + if (rc < 0) { + rc = FW_UPLOAD_ERR_RW_ERROR; + goto out_free; + } + + *written = cur_size; + + /* Activate FW if oneshot or if the last slice was written */ + if (mds->fw.oneshot || remaining == 0) { + dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", + mds->fw.next_slot); + rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); + if (rc < 0) { + dev_err(&cxlmd->dev, "Error activating firmware: %d\n", + rc); + rc = FW_UPLOAD_ERR_HW_ERROR; + goto out_free; + } + } + + rc = FW_UPLOAD_ERR_NONE; + +out_free: + kfree(transfer); + return rc; +} + +static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) +{ + struct cxl_memdev_state *mds = fwl->dd_handle; + + /* + * cxl_internal_send_cmd() handles background operations synchronously. + * No need to wait for completions here - any errors would've been + * reported and handled during the ->write() call(s). + * Just check if a cancel request was received, and return success. + */ + if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) + return cxl_fw_do_cancel(fwl); + + return FW_UPLOAD_ERR_NONE; +} + +static void cxl_fw_cancel(struct fw_upload *fwl) +{ + struct cxl_memdev_state *mds = fwl->dd_handle; + + set_bit(CXL_FW_CANCEL, mds->fw.state); +} + +static const struct fw_upload_ops cxl_memdev_fw_ops = { + .prepare = cxl_fw_prepare, + .write = cxl_fw_write, + .poll_complete = cxl_fw_poll_complete, + .cancel = cxl_fw_cancel, + .cleanup = cxl_fw_cleanup, +}; + +static void devm_cxl_remove_fw_upload(void *fwl) +{ + firmware_upload_unregister(fwl); +} + +int cxl_memdev_setup_fw_upload(struct cxl_memdev_state *mds) +{ + struct cxl_dev_state *cxlds = &mds->cxlds; + struct device *dev = &cxlds->cxlmd->dev; + struct fw_upload *fwl; + int rc; + + if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds)) + return 0; + + fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), + &cxl_memdev_fw_ops, mds); + if (IS_ERR(fwl)) + return dev_err_probe(dev, PTR_ERR(fwl), + "Failed to register firmware loader\n"); + + rc = devm_add_action_or_reset(cxlds->dev, devm_cxl_remove_fw_upload, + fwl); + if (rc) + dev_err(dev, + "Failed to add firmware loader remove action: %d\n", + rc); + + return rc; +} +EXPORT_SYMBOL_NS_GPL(cxl_memdev_setup_fw_upload, CXL); + static const struct file_operations cxl_memdev_fops = { .owner = THIS_MODULE, .unlocked_ioctl = cxl_memdev_ioctl, @@ -551,6 +983,35 @@ static const struct file_operations cxl_memdev_fops = { .llseek = noop_llseek, }; +static void put_sanitize(void *data) +{ + struct cxl_memdev_state *mds = data; + + sysfs_put(mds->security.sanitize_node); +} + +static int cxl_memdev_security_init(struct cxl_memdev *cxlmd) +{ + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct device *dev = &cxlmd->dev; + struct kernfs_node *sec; + + sec = sysfs_get_dirent(dev->kobj.sd, "security"); + if (!sec) { + dev_err(dev, "sysfs_get_dirent 'security' failed\n"); + return -ENODEV; + } + mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); + sysfs_put(sec); + if (!mds->security.sanitize_node) { + dev_err(dev, "sysfs_get_dirent 'state' failed\n"); + return -ENODEV; + } + + return devm_add_action_or_reset(cxlds->dev, put_sanitize, mds); + } + struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds) { struct cxl_memdev *cxlmd; @@ -579,6 +1040,10 @@ struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds) if (rc) goto err; + rc = cxl_memdev_security_init(cxlmd); + if (rc) + goto err; + rc = devm_add_action_or_reset(cxlds->dev, cxl_memdev_unregister, cxlmd); if (rc) return ERR_PTR(rc); diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 67f4ab6daa34..c7a7887ebdcf 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -67,7 +67,7 @@ static int match_add_dports(struct pci_dev *pdev, void *data) /** * devm_cxl_port_enumerate_dports - enumerate downstream ports of the upstream port - * @port: cxl_port whose ->uport is the upstream of dports to be enumerated + * @port: cxl_port whose ->uport_dev is the upstream of dports to be enumerated * * Returns a positive number of dports enumerated or a negative error * code. @@ -308,36 +308,17 @@ static void disable_hdm(void *_cxlhdm) hdm + CXL_HDM_DECODER_CTRL_OFFSET); } -int devm_cxl_enable_hdm(struct cxl_port *port, struct cxl_hdm *cxlhdm) +static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm) { - void __iomem *hdm; + void __iomem *hdm = cxlhdm->regs.hdm_decoder; u32 global_ctrl; - /* - * If the hdm capability was not mapped there is nothing to enable and - * the caller is responsible for what happens next. For example, - * emulate a passthrough decoder. - */ - if (IS_ERR(cxlhdm)) - return 0; - - hdm = cxlhdm->regs.hdm_decoder; global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); - - /* - * If the HDM decoder capability was enabled on entry, skip - * registering disable_hdm() since this decode capability may be - * owned by platform firmware. - */ - if (global_ctrl & CXL_HDM_DECODER_ENABLE) - return 0; - writel(global_ctrl | CXL_HDM_DECODER_ENABLE, hdm + CXL_HDM_DECODER_CTRL_OFFSET); - return devm_add_action_or_reset(&port->dev, disable_hdm, cxlhdm); + return devm_add_action_or_reset(host, disable_hdm, cxlhdm); } -EXPORT_SYMBOL_NS_GPL(devm_cxl_enable_hdm, CXL); int cxl_dvsec_rr_decode(struct device *dev, int d, struct cxl_endpoint_dvsec_info *info) @@ -511,7 +492,7 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm, if (info->mem_enabled) return 0; - rc = devm_cxl_enable_hdm(port, cxlhdm); + rc = devm_cxl_enable_hdm(&port->dev, cxlhdm); if (rc) return rc; @@ -622,7 +603,7 @@ static int cxl_cdat_read_table(struct device *dev, */ void read_cdat_data(struct cxl_port *port) { - struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport); + struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); struct device *host = cxlmd->dev.parent; struct device *dev = &port->dev; struct pci_doe_mb *cdat_doe; diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c index f8c38d997252..fc94f5240327 100644 --- a/drivers/cxl/core/pmem.c +++ b/drivers/cxl/core/pmem.c @@ -64,7 +64,7 @@ static int match_nvdimm_bridge(struct device *dev, void *data) struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_memdev *cxlmd) { - struct cxl_port *port = find_cxl_root(dev_get_drvdata(&cxlmd->dev)); + struct cxl_port *port = find_cxl_root(cxlmd->endpoint); struct device *dev; if (!port) diff --git a/drivers/cxl/core/pmu.c b/drivers/cxl/core/pmu.c new file mode 100644 index 000000000000..7684c843e5a5 --- /dev/null +++ b/drivers/cxl/core/pmu.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Huawei. All rights reserved. */ + +#include <linux/device.h> +#include <linux/slab.h> +#include <linux/idr.h> +#include <cxlmem.h> +#include <pmu.h> +#include <cxl.h> +#include "core.h" + +static void cxl_pmu_release(struct device *dev) +{ + struct cxl_pmu *pmu = to_cxl_pmu(dev); + + kfree(pmu); +} + +const struct device_type cxl_pmu_type = { + .name = "cxl_pmu", + .release = cxl_pmu_release, +}; + +static void remove_dev(void *dev) +{ + device_del(dev); +} + +int devm_cxl_pmu_add(struct device *parent, struct cxl_pmu_regs *regs, + int assoc_id, int index, enum cxl_pmu_type type) +{ + struct cxl_pmu *pmu; + struct device *dev; + int rc; + + pmu = kzalloc(sizeof(*pmu), GFP_KERNEL); + if (!pmu) + return -ENOMEM; + + pmu->assoc_id = assoc_id; + pmu->index = index; + pmu->type = type; + pmu->base = regs->pmu; + dev = &pmu->dev; + device_initialize(dev); + device_set_pm_not_required(dev); + dev->parent = parent; + dev->bus = &cxl_bus_type; + dev->type = &cxl_pmu_type; + switch (pmu->type) { + case CXL_PMU_MEMDEV: + rc = dev_set_name(dev, "pmu_mem%d.%d", assoc_id, index); + break; + } + if (rc) + goto err; + + rc = device_add(dev); + if (rc) + goto err; + + return devm_add_action_or_reset(parent, remove_dev, dev); + +err: + put_device(&pmu->dev); + return rc; +} +EXPORT_SYMBOL_NS_GPL(devm_cxl_pmu_add, CXL); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index e7c284c890bc..724be8448eb4 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -56,6 +56,8 @@ static int cxl_device_id(const struct device *dev) return CXL_DEVICE_MEMORY_EXPANDER; if (dev->type == CXL_REGION_TYPE()) return CXL_DEVICE_REGION; + if (dev->type == &cxl_pmu_type) + return CXL_DEVICE_PMU; return 0; } @@ -117,9 +119,9 @@ static ssize_t target_type_show(struct device *dev, struct cxl_decoder *cxld = to_cxl_decoder(dev); switch (cxld->target_type) { - case CXL_DECODER_ACCELERATOR: + case CXL_DECODER_DEVMEM: return sysfs_emit(buf, "accelerator\n"); - case CXL_DECODER_EXPANDER: + case CXL_DECODER_HOSTONLYMEM: return sysfs_emit(buf, "expander\n"); } return -ENXIO; @@ -561,9 +563,9 @@ static void unregister_port(void *_port) * unregistered while holding their parent port lock. */ if (!parent) - lock_dev = port->uport; + lock_dev = port->uport_dev; else if (is_cxl_root(parent)) - lock_dev = parent->uport; + lock_dev = parent->uport_dev; else lock_dev = &parent->dev; @@ -583,7 +585,8 @@ static int devm_cxl_link_uport(struct device *host, struct cxl_port *port) { int rc; - rc = sysfs_create_link(&port->dev.kobj, &port->uport->kobj, "uport"); + rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj, + "uport"); if (rc) return rc; return devm_add_action_or_reset(host, cxl_unlink_uport, port); @@ -605,7 +608,7 @@ static int devm_cxl_link_parent_dport(struct device *host, if (!parent_dport) return 0; - rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport->kobj, + rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj, "parent_dport"); if (rc) return rc; @@ -614,7 +617,7 @@ static int devm_cxl_link_parent_dport(struct device *host, static struct lock_class_key cxl_port_key; -static struct cxl_port *cxl_port_alloc(struct device *uport, +static struct cxl_port *cxl_port_alloc(struct device *uport_dev, resource_size_t component_reg_phys, struct cxl_dport *parent_dport) { @@ -630,7 +633,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport, if (rc < 0) goto err; port->id = rc; - port->uport = uport; + port->uport_dev = uport_dev; /* * The top-level cxl_port "cxl_root" does not have a cxl_port as @@ -658,12 +661,13 @@ static struct cxl_port *cxl_port_alloc(struct device *uport, if (iter->host_bridge) port->host_bridge = iter->host_bridge; else if (parent_dport->rch) - port->host_bridge = parent_dport->dport; + port->host_bridge = parent_dport->dport_dev; else - port->host_bridge = iter->uport; - dev_dbg(uport, "host-bridge: %s\n", dev_name(port->host_bridge)); + port->host_bridge = iter->uport_dev; + dev_dbg(uport_dev, "host-bridge: %s\n", + dev_name(port->host_bridge)); } else - dev->parent = uport; + dev->parent = uport_dev; port->component_reg_phys = component_reg_phys; ida_init(&port->decoder_ida); @@ -686,8 +690,38 @@ err: return ERR_PTR(rc); } +static int cxl_setup_comp_regs(struct device *dev, struct cxl_register_map *map, + resource_size_t component_reg_phys) +{ + if (component_reg_phys == CXL_RESOURCE_NONE) + return 0; + + *map = (struct cxl_register_map) { + .dev = dev, + .reg_type = CXL_REGLOC_RBI_COMPONENT, + .resource = component_reg_phys, + .max_size = CXL_COMPONENT_REG_BLOCK_SIZE, + }; + + return cxl_setup_regs(map); +} + +static inline int cxl_port_setup_regs(struct cxl_port *port, + resource_size_t component_reg_phys) +{ + return cxl_setup_comp_regs(&port->dev, &port->comp_map, + component_reg_phys); +} + +static inline int cxl_dport_setup_regs(struct cxl_dport *dport, + resource_size_t component_reg_phys) +{ + return cxl_setup_comp_regs(dport->dport_dev, &dport->comp_map, + component_reg_phys); +} + static struct cxl_port *__devm_cxl_add_port(struct device *host, - struct device *uport, + struct device *uport_dev, resource_size_t component_reg_phys, struct cxl_dport *parent_dport) { @@ -695,12 +729,12 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, struct device *dev; int rc; - port = cxl_port_alloc(uport, component_reg_phys, parent_dport); + port = cxl_port_alloc(uport_dev, component_reg_phys, parent_dport); if (IS_ERR(port)) return port; dev = &port->dev; - if (is_cxl_memdev(uport)) + if (is_cxl_memdev(uport_dev)) rc = dev_set_name(dev, "endpoint%d", port->id); else if (parent_dport) rc = dev_set_name(dev, "port%d", port->id); @@ -709,6 +743,10 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, if (rc) goto err; + rc = cxl_port_setup_regs(port, component_reg_phys); + if (rc) + goto err; + rc = device_add(dev); if (rc) goto err; @@ -735,28 +773,29 @@ err: /** * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy * @host: host device for devm operations - * @uport: "physical" device implementing this upstream port + * @uport_dev: "physical" device implementing this upstream port * @component_reg_phys: (optional) for configurable cxl_port instances * @parent_dport: next hop up in the CXL memory decode hierarchy */ -struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport, +struct cxl_port *devm_cxl_add_port(struct device *host, + struct device *uport_dev, resource_size_t component_reg_phys, struct cxl_dport *parent_dport) { struct cxl_port *port, *parent_port; - port = __devm_cxl_add_port(host, uport, component_reg_phys, + port = __devm_cxl_add_port(host, uport_dev, component_reg_phys, parent_dport); parent_port = parent_dport ? parent_dport->port : NULL; if (IS_ERR(port)) { - dev_dbg(uport, "Failed to add%s%s%s: %ld\n", + dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n", parent_port ? " port to " : "", parent_port ? dev_name(&parent_port->dev) : "", parent_port ? "" : " root port", PTR_ERR(port)); } else { - dev_dbg(uport, "%s added%s%s%s\n", + dev_dbg(uport_dev, "%s added%s%s%s\n", dev_name(&port->dev), parent_port ? " to " : "", parent_port ? dev_name(&parent_port->dev) : "", @@ -773,33 +812,34 @@ struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port) if (is_cxl_root(port)) return NULL; - if (dev_is_pci(port->uport)) { - struct pci_dev *pdev = to_pci_dev(port->uport); + if (dev_is_pci(port->uport_dev)) { + struct pci_dev *pdev = to_pci_dev(port->uport_dev); return pdev->subordinate; } - return xa_load(&cxl_root_buses, (unsigned long)port->uport); + return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev); } EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL); -static void unregister_pci_bus(void *uport) +static void unregister_pci_bus(void *uport_dev) { - xa_erase(&cxl_root_buses, (unsigned long)uport); + xa_erase(&cxl_root_buses, (unsigned long)uport_dev); } -int devm_cxl_register_pci_bus(struct device *host, struct device *uport, +int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev, struct pci_bus *bus) { int rc; - if (dev_is_pci(uport)) + if (dev_is_pci(uport_dev)) return -EINVAL; - rc = xa_insert(&cxl_root_buses, (unsigned long)uport, bus, GFP_KERNEL); + rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus, + GFP_KERNEL); if (rc) return rc; - return devm_add_action_or_reset(host, unregister_pci_bus, uport); + return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev); } EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL); @@ -847,22 +887,22 @@ static struct cxl_dport *find_dport(struct cxl_port *port, int id) return NULL; } -static int add_dport(struct cxl_port *port, struct cxl_dport *new) +static int add_dport(struct cxl_port *port, struct cxl_dport *dport) { struct cxl_dport *dup; int rc; device_lock_assert(&port->dev); - dup = find_dport(port, new->port_id); + dup = find_dport(port, dport->port_id); if (dup) { dev_err(&port->dev, "unable to add dport%d-%s non-unique port id (%s)\n", - new->port_id, dev_name(new->dport), - dev_name(dup->dport)); + dport->port_id, dev_name(dport->dport_dev), + dev_name(dup->dport_dev)); return -EBUSY; } - rc = xa_insert(&port->dports, (unsigned long)new->dport, new, + rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport, GFP_KERNEL); if (rc) return rc; @@ -895,8 +935,8 @@ static void cxl_dport_remove(void *data) struct cxl_dport *dport = data; struct cxl_port *port = dport->port; - xa_erase(&port->dports, (unsigned long) dport->dport); - put_device(dport->dport); + xa_erase(&port->dports, (unsigned long) dport->dport_dev); + put_device(dport->dport_dev); } static void cxl_dport_unlink(void *data) @@ -920,7 +960,7 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, int rc; if (is_cxl_root(port)) - host = port->uport; + host = port->uport_dev; else host = &port->dev; @@ -938,13 +978,29 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, if (!dport) return ERR_PTR(-ENOMEM); - dport->dport = dport_dev; + if (rcrb != CXL_RESOURCE_NONE) { + dport->rcrb.base = rcrb; + component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb, + CXL_RCRB_DOWNSTREAM); + if (component_reg_phys == CXL_RESOURCE_NONE) { + dev_warn(dport_dev, "Invalid Component Registers in RCRB"); + return ERR_PTR(-ENXIO); + } + + dport->rch = true; + } + + if (component_reg_phys != CXL_RESOURCE_NONE) + dev_dbg(dport_dev, "Component Registers found for dport: %pa\n", + &component_reg_phys); + + dport->dport_dev = dport_dev; dport->port_id = port_id; - dport->component_reg_phys = component_reg_phys; dport->port = port; - if (rcrb != CXL_RESOURCE_NONE) - dport->rch = true; - dport->rcrb = rcrb; + + rc = cxl_dport_setup_regs(dport, component_reg_phys); + if (rc) + return ERR_PTR(rc); cond_cxl_root_lock(port); rc = add_dport(port, dport); @@ -1004,14 +1060,12 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL); * @port: the cxl_port that references this dport * @dport_dev: firmware or PCI device representing the dport * @port_id: identifier for this dport in a decoder's target list - * @component_reg_phys: optional location of CXL component registers * @rcrb: mandatory location of a Root Complex Register Block * * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH */ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, struct device *dport_dev, int port_id, - resource_size_t component_reg_phys, resource_size_t rcrb) { struct cxl_dport *dport; @@ -1022,7 +1076,7 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, } dport = __devm_cxl_add_dport(port, dport_dev, port_id, - component_reg_phys, rcrb); + CXL_RESOURCE_NONE, rcrb); if (IS_ERR(dport)) { dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n", dev_name(&port->dev), PTR_ERR(dport)); @@ -1161,7 +1215,7 @@ static struct device *grandparent(struct device *dev) static void delete_endpoint(void *data) { struct cxl_memdev *cxlmd = data; - struct cxl_port *endpoint = dev_get_drvdata(&cxlmd->dev); + struct cxl_port *endpoint = cxlmd->endpoint; struct cxl_port *parent_port; struct device *parent; @@ -1176,6 +1230,7 @@ static void delete_endpoint(void *data) devm_release_action(parent, cxl_unlink_uport, endpoint); devm_release_action(parent, unregister_port, endpoint); } + cxlmd->endpoint = NULL; device_unlock(parent); put_device(parent); out: @@ -1187,7 +1242,7 @@ int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint) struct device *dev = &cxlmd->dev; get_device(&endpoint->dev); - dev_set_drvdata(dev, endpoint); + cxlmd->endpoint = endpoint; cxlmd->depth = endpoint->depth; return devm_add_action_or_reset(dev, delete_endpoint, cxlmd); } @@ -1363,7 +1418,7 @@ out: rc = PTR_ERR(port); else { dev_dbg(&cxlmd->dev, "add to new port %s:%s\n", - dev_name(&port->dev), dev_name(port->uport)); + dev_name(&port->dev), dev_name(port->uport_dev)); rc = cxl_add_ep(dport, &cxlmd->dev); if (rc == -EBUSY) { /* @@ -1425,7 +1480,8 @@ retry: if (port) { dev_dbg(&cxlmd->dev, "found already registered port %s:%s\n", - dev_name(&port->dev), dev_name(port->uport)); + dev_name(&port->dev), + dev_name(port->uport_dev)); rc = cxl_add_ep(dport, &cxlmd->dev); /* @@ -1465,6 +1521,13 @@ retry: } EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL); +struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev, + struct cxl_dport **dport) +{ + return find_cxl_port(pdev->dev.parent, dport); +} +EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, CXL); + struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd, struct cxl_dport **dport) { @@ -1550,7 +1613,7 @@ static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld) /* Pre initialize an "empty" decoder */ cxld->interleave_ways = 1; cxld->interleave_granularity = PAGE_SIZE; - cxld->target_type = CXL_DECODER_EXPANDER; + cxld->target_type = CXL_DECODER_HOSTONLYMEM; cxld->hpa_range = (struct range) { .start = 0, .end = -1, diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index f822de44bee0..e115ba382e04 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -125,10 +125,38 @@ static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port, return xa_load(&port->regions, (unsigned long)cxlr); } +static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) +{ + if (!cpu_cache_has_invalidate_memregion()) { + if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) { + dev_warn_once( + &cxlr->dev, + "Bypassing cpu_cache_invalidate_memregion() for testing!\n"); + return 0; + } else { + dev_err(&cxlr->dev, + "Failed to synchronize CPU cache state\n"); + return -ENXIO; + } + } + + cpu_cache_invalidate_memregion(IORES_DESC_CXL); + return 0; +} + static int cxl_region_decode_reset(struct cxl_region *cxlr, int count) { struct cxl_region_params *p = &cxlr->params; - int i; + int i, rc = 0; + + /* + * Before region teardown attempt to flush, and if the flush + * fails cancel the region teardown for data consistency + * concerns + */ + rc = cxl_region_invalidate_memregion(cxlr); + if (rc) + return rc; for (i = count - 1; i >= 0; i--) { struct cxl_endpoint_decoder *cxled = p->targets[i]; @@ -136,7 +164,6 @@ static int cxl_region_decode_reset(struct cxl_region *cxlr, int count) struct cxl_port *iter = cxled_to_port(cxled); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_ep *ep; - int rc = 0; if (cxlds->rcd) goto endpoint_reset; @@ -155,14 +182,19 @@ static int cxl_region_decode_reset(struct cxl_region *cxlr, int count) rc = cxld->reset(cxld); if (rc) return rc; + set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); } endpoint_reset: rc = cxled->cxld.reset(&cxled->cxld); if (rc) return rc; + set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); } + /* all decoders associated with this region have been torn down */ + clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); + return 0; } @@ -256,9 +288,19 @@ static ssize_t commit_store(struct device *dev, struct device_attribute *attr, goto out; } - if (commit) + /* + * Invalidate caches before region setup to drop any speculative + * consumption of this address space + */ + rc = cxl_region_invalidate_memregion(cxlr); + if (rc) + return rc; + + if (commit) { rc = cxl_region_decode_commit(cxlr); - else { + if (rc == 0) + p->state = CXL_CONFIG_COMMIT; + } else { p->state = CXL_CONFIG_RESET_PENDING; up_write(&cxl_region_rwsem); device_release_driver(&cxlr->dev); @@ -268,18 +310,20 @@ static ssize_t commit_store(struct device *dev, struct device_attribute *attr, * The lock was dropped, so need to revalidate that the reset is * still pending. */ - if (p->state == CXL_CONFIG_RESET_PENDING) + if (p->state == CXL_CONFIG_RESET_PENDING) { rc = cxl_region_decode_reset(cxlr, p->interleave_ways); + /* + * Revert to committed since there may still be active + * decoders associated with this region, or move forward + * to active to mark the reset successful + */ + if (rc) + p->state = CXL_CONFIG_COMMIT; + else + p->state = CXL_CONFIG_ACTIVE; + } } - if (rc) - goto out; - - if (commit) - p->state = CXL_CONFIG_COMMIT; - else if (p->state == CXL_CONFIG_RESET_PENDING) - p->state = CXL_CONFIG_ACTIVE; - out: up_write(&cxl_region_rwsem); @@ -809,6 +853,18 @@ static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr, return -EBUSY; } + /* + * Endpoints should already match the region type, but backstop that + * assumption with an assertion. Switch-decoders change mapping-type + * based on what is mapped when they are assigned to a region. + */ + dev_WARN_ONCE(&cxlr->dev, + port == cxled_to_port(cxled) && + cxld->target_type != cxlr->type, + "%s:%s mismatch decoder type %d -> %d\n", + dev_name(&cxled_to_memdev(cxled)->dev), + dev_name(&cxld->dev), cxld->target_type, cxlr->type); + cxld->target_type = cxlr->type; cxl_rr->decoder = cxld; return 0; } @@ -906,10 +962,10 @@ static int cxl_port_attach_region(struct cxl_port *port, dev_dbg(&cxlr->dev, "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n", - dev_name(port->uport), dev_name(&port->dev), + dev_name(port->uport_dev), dev_name(&port->dev), dev_name(&cxld->dev), dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos, - ep ? ep->next ? dev_name(ep->next->uport) : + ep ? ep->next ? dev_name(ep->next->uport_dev) : dev_name(&cxlmd->dev) : "none", cxl_rr->nr_eps, cxl_rr->nr_targets); @@ -984,7 +1040,7 @@ static int check_last_peer(struct cxl_endpoint_decoder *cxled, */ if (pos < distance) { dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n", - dev_name(port->uport), dev_name(&port->dev), + dev_name(port->uport_dev), dev_name(&port->dev), dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); return -ENXIO; } @@ -994,7 +1050,7 @@ static int check_last_peer(struct cxl_endpoint_decoder *cxled, if (ep->dport != ep_peer->dport) { dev_dbg(&cxlr->dev, "%s:%s: %s:%s pos %d mismatched peer %s:%s\n", - dev_name(port->uport), dev_name(&port->dev), + dev_name(port->uport_dev), dev_name(&port->dev), dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos, dev_name(&cxlmd_peer->dev), dev_name(&cxled_peer->cxld.dev)); @@ -1026,7 +1082,7 @@ static int cxl_port_setup_targets(struct cxl_port *port, */ if (!is_power_of_2(cxl_rr->nr_targets)) { dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n", - dev_name(port->uport), dev_name(&port->dev), + dev_name(port->uport_dev), dev_name(&port->dev), cxl_rr->nr_targets); return -EINVAL; } @@ -1076,7 +1132,7 @@ static int cxl_port_setup_targets(struct cxl_port *port, rc = granularity_to_eig(parent_ig, &peig); if (rc) { dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n", - dev_name(parent_port->uport), + dev_name(parent_port->uport_dev), dev_name(&parent_port->dev), parent_ig); return rc; } @@ -1084,7 +1140,7 @@ static int cxl_port_setup_targets(struct cxl_port *port, rc = ways_to_eiw(parent_iw, &peiw); if (rc) { dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n", - dev_name(parent_port->uport), + dev_name(parent_port->uport_dev), dev_name(&parent_port->dev), parent_iw); return rc; } @@ -1093,7 +1149,7 @@ static int cxl_port_setup_targets(struct cxl_port *port, rc = ways_to_eiw(iw, &eiw); if (rc) { dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n", - dev_name(port->uport), dev_name(&port->dev), iw); + dev_name(port->uport_dev), dev_name(&port->dev), iw); return rc; } @@ -1113,7 +1169,7 @@ static int cxl_port_setup_targets(struct cxl_port *port, rc = eig_to_granularity(eig, &ig); if (rc) { dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n", - dev_name(port->uport), dev_name(&port->dev), + dev_name(port->uport_dev), dev_name(&port->dev), 256 << eig); return rc; } @@ -1126,11 +1182,11 @@ static int cxl_port_setup_targets(struct cxl_port *port, ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) { dev_err(&cxlr->dev, "%s:%s %s expected iw: %d ig: %d %pr\n", - dev_name(port->uport), dev_name(&port->dev), + dev_name(port->uport_dev), dev_name(&port->dev), __func__, iw, ig, p->res); dev_err(&cxlr->dev, "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n", - dev_name(port->uport), dev_name(&port->dev), + dev_name(port->uport_dev), dev_name(&port->dev), __func__, cxld->interleave_ways, cxld->interleave_granularity, (cxld->flags & CXL_DECODER_F_ENABLE) ? @@ -1147,22 +1203,22 @@ static int cxl_port_setup_targets(struct cxl_port *port, .end = p->res->end, }; } - dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport), + dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev), dev_name(&port->dev), iw, ig); add_target: if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) { dev_dbg(&cxlr->dev, "%s:%s: targets full trying to add %s:%s at %d\n", - dev_name(port->uport), dev_name(&port->dev), + dev_name(port->uport_dev), dev_name(&port->dev), dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); return -ENXIO; } if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) { dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n", - dev_name(port->uport), dev_name(&port->dev), + dev_name(port->uport_dev), dev_name(&port->dev), dev_name(&cxlsd->cxld.dev), - dev_name(ep->dport->dport), + dev_name(ep->dport->dport_dev), cxl_rr->nr_targets_set); return -ENXIO; } @@ -1172,8 +1228,8 @@ add_target: out_target_set: cxl_rr->nr_targets_set += inc; dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n", - dev_name(port->uport), dev_name(&port->dev), - cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport), + dev_name(port->uport_dev), dev_name(&port->dev), + cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev), dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); return 0; @@ -1492,7 +1548,7 @@ static int cmp_decode_pos(const void *a, const void *b) if (!dev) { struct range *range = &cxled_a->cxld.hpa_range; - dev_err(port->uport, + dev_err(port->uport_dev, "failed to find decoder that maps %#llx-%#llx\n", range->start, range->end); goto err; @@ -1507,14 +1563,15 @@ static int cmp_decode_pos(const void *a, const void *b) put_device(dev); if (a_pos < 0 || b_pos < 0) { - dev_err(port->uport, + dev_err(port->uport_dev, "failed to find shared decoder for %s and %s\n", dev_name(cxlmd_a->dev.parent), dev_name(cxlmd_b->dev.parent)); goto err; } - dev_dbg(port->uport, "%s comes %s %s\n", dev_name(cxlmd_a->dev.parent), + dev_dbg(port->uport_dev, "%s comes %s %s\n", + dev_name(cxlmd_a->dev.parent), a_pos - b_pos < 0 ? "before" : "after", dev_name(cxlmd_b->dev.parent)); @@ -1674,7 +1731,6 @@ static int cxl_region_attach(struct cxl_region *cxlr, if (rc) goto err_decrement; p->state = CXL_CONFIG_ACTIVE; - set_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags); } cxled->cxld.interleave_ways = p->interleave_ways; @@ -2059,11 +2115,11 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, if (rc) goto err; - rc = devm_add_action_or_reset(port->uport, unregister_region, cxlr); + rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); if (rc) return ERR_PTR(rc); - dev_dbg(port->uport, "%s: created %s\n", + dev_dbg(port->uport_dev, "%s: created %s\n", dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); return cxlr; @@ -2103,7 +2159,7 @@ static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd, return ERR_PTR(-EBUSY); } - return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_EXPANDER); + return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM); } static ssize_t create_pmem_region_store(struct device *dev, @@ -2191,7 +2247,7 @@ static ssize_t delete_region_store(struct device *dev, if (IS_ERR(cxlr)) return PTR_ERR(cxlr); - devm_release_action(port->uport, unregister_region, cxlr); + devm_release_action(port->uport_dev, unregister_region, cxlr); put_device(&cxlr->dev); return len; @@ -2356,7 +2412,8 @@ int cxl_get_poison_by_endpoint(struct cxl_port *port) rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder); if (rc == 1) - rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport), &ctx); + rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev), + &ctx); up_read(&cxl_region_rwsem); return rc; @@ -2732,7 +2789,7 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd, err: up_write(&cxl_region_rwsem); - devm_release_action(port->uport, unregister_region, cxlr); + devm_release_action(port->uport_dev, unregister_region, cxlr); return ERR_PTR(rc); } @@ -2803,30 +2860,6 @@ out: } EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL); -static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) -{ - if (!test_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags)) - return 0; - - if (!cpu_cache_has_invalidate_memregion()) { - if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) { - dev_warn_once( - &cxlr->dev, - "Bypassing cpu_cache_invalidate_memregion() for testing!\n"); - clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags); - return 0; - } else { - dev_err(&cxlr->dev, - "Failed to synchronize CPU cache state\n"); - return -ENXIO; - } - } - - cpu_cache_invalidate_memregion(IORES_DESC_CXL); - clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags); - return 0; -} - static int is_system_ram(struct resource *res, void *arg) { struct cxl_region *cxlr = arg; @@ -2854,7 +2887,12 @@ static int cxl_region_probe(struct device *dev) goto out; } - rc = cxl_region_invalidate_memregion(cxlr); + if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) { + dev_err(&cxlr->dev, + "failed to activate, re-commit region and retry\n"); + rc = -ENXIO; + goto out; + } /* * From this point on any path that changes the region's state away from diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c index 1476a0299c9b..6281127b3e9d 100644 --- a/drivers/cxl/core/regs.c +++ b/drivers/cxl/core/regs.c @@ -6,6 +6,7 @@ #include <linux/pci.h> #include <cxlmem.h> #include <cxlpci.h> +#include <pmu.h> #include "core.h" @@ -199,11 +200,13 @@ void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr, return ret_val; } -int cxl_map_component_regs(struct device *dev, struct cxl_component_regs *regs, - struct cxl_register_map *map, unsigned long map_mask) +int cxl_map_component_regs(const struct cxl_register_map *map, + struct cxl_component_regs *regs, + unsigned long map_mask) { + struct device *dev = map->dev; struct mapinfo { - struct cxl_reg_map *rmap; + const struct cxl_reg_map *rmap; void __iomem **addr; } mapinfo[] = { { &map->component_map.hdm_decoder, ®s->hdm_decoder }, @@ -231,13 +234,13 @@ int cxl_map_component_regs(struct device *dev, struct cxl_component_regs *regs, } EXPORT_SYMBOL_NS_GPL(cxl_map_component_regs, CXL); -int cxl_map_device_regs(struct device *dev, - struct cxl_device_regs *regs, - struct cxl_register_map *map) +int cxl_map_device_regs(const struct cxl_register_map *map, + struct cxl_device_regs *regs) { + struct device *dev = map->dev; resource_size_t phys_addr = map->resource; struct mapinfo { - struct cxl_reg_map *rmap; + const struct cxl_reg_map *rmap; void __iomem **addr; } mapinfo[] = { { &map->device_map.status, ®s->status, }, @@ -286,23 +289,30 @@ static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi, } /** - * cxl_find_regblock() - Locate register blocks by type + * cxl_find_regblock_instance() - Locate a register block by type / index * @pdev: The CXL PCI device to enumerate. * @type: Register Block Indicator id * @map: Enumeration output, clobbered on error + * @index: Index into which particular instance of a regblock wanted in the + * order found in register locator DVSEC. * * Return: 0 if register block enumerated, negative error code otherwise * * A CXL DVSEC may point to one or more register blocks, search for them - * by @type. + * by @type and @index. */ -int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type, - struct cxl_register_map *map) +int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type, + struct cxl_register_map *map, int index) { u32 regloc_size, regblocks; + int instance = 0; int regloc, i; - map->resource = CXL_RESOURCE_NONE; + *map = (struct cxl_register_map) { + .dev = &pdev->dev, + .resource = CXL_RESOURCE_NONE, + }; + regloc = pci_find_dvsec_capability(pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DVSEC_REG_LOCATOR); if (!regloc) @@ -323,20 +333,148 @@ int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type, if (!cxl_decode_regblock(pdev, reg_lo, reg_hi, map)) continue; - if (map->reg_type == type) - return 0; + if (map->reg_type == type) { + if (index == instance) + return 0; + instance++; + } } map->resource = CXL_RESOURCE_NONE; return -ENODEV; } +EXPORT_SYMBOL_NS_GPL(cxl_find_regblock_instance, CXL); + +/** + * cxl_find_regblock() - Locate register blocks by type + * @pdev: The CXL PCI device to enumerate. + * @type: Register Block Indicator id + * @map: Enumeration output, clobbered on error + * + * Return: 0 if register block enumerated, negative error code otherwise + * + * A CXL DVSEC may point to one or more register blocks, search for them + * by @type. + */ +int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type, + struct cxl_register_map *map) +{ + return cxl_find_regblock_instance(pdev, type, map, 0); +} EXPORT_SYMBOL_NS_GPL(cxl_find_regblock, CXL); -resource_size_t cxl_rcrb_to_component(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which) +/** + * cxl_count_regblock() - Count instances of a given regblock type. + * @pdev: The CXL PCI device to enumerate. + * @type: Register Block Indicator id + * + * Some regblocks may be repeated. Count how many instances. + * + * Return: count of matching regblocks. + */ +int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type) +{ + struct cxl_register_map map; + int rc, count = 0; + + while (1) { + rc = cxl_find_regblock_instance(pdev, type, &map, count); + if (rc) + return count; + count++; + } +} +EXPORT_SYMBOL_NS_GPL(cxl_count_regblock, CXL); + +int cxl_map_pmu_regs(struct pci_dev *pdev, struct cxl_pmu_regs *regs, + struct cxl_register_map *map) +{ + struct device *dev = &pdev->dev; + resource_size_t phys_addr; + + phys_addr = map->resource; + regs->pmu = devm_cxl_iomap_block(dev, phys_addr, CXL_PMU_REGMAP_SIZE); + if (!regs->pmu) + return -ENOMEM; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_map_pmu_regs, CXL); + +static int cxl_map_regblock(struct cxl_register_map *map) +{ + struct device *dev = map->dev; + + map->base = ioremap(map->resource, map->max_size); + if (!map->base) { + dev_err(dev, "failed to map registers\n"); + return -ENOMEM; + } + + dev_dbg(dev, "Mapped CXL Memory Device resource %pa\n", &map->resource); + return 0; +} + +static void cxl_unmap_regblock(struct cxl_register_map *map) +{ + iounmap(map->base); + map->base = NULL; +} + +static int cxl_probe_regs(struct cxl_register_map *map) +{ + struct cxl_component_reg_map *comp_map; + struct cxl_device_reg_map *dev_map; + struct device *dev = map->dev; + void __iomem *base = map->base; + + switch (map->reg_type) { + case CXL_REGLOC_RBI_COMPONENT: + comp_map = &map->component_map; + cxl_probe_component_regs(dev, base, comp_map); + dev_dbg(dev, "Set up component registers\n"); + break; + case CXL_REGLOC_RBI_MEMDEV: + dev_map = &map->device_map; + cxl_probe_device_regs(dev, base, dev_map); + if (!dev_map->status.valid || !dev_map->mbox.valid || + !dev_map->memdev.valid) { + dev_err(dev, "registers not found: %s%s%s\n", + !dev_map->status.valid ? "status " : "", + !dev_map->mbox.valid ? "mbox " : "", + !dev_map->memdev.valid ? "memdev " : ""); + return -ENXIO; + } + + dev_dbg(dev, "Probing device registers...\n"); + break; + default: + break; + } + + return 0; +} + +int cxl_setup_regs(struct cxl_register_map *map) +{ + int rc; + + rc = cxl_map_regblock(map); + if (rc) + return rc; + + rc = cxl_probe_regs(map); + cxl_unmap_regblock(map); + + return rc; +} +EXPORT_SYMBOL_NS_GPL(cxl_setup_regs, CXL); + +resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri, + enum cxl_rcrb which) { resource_size_t component_reg_phys; + resource_size_t rcrb = ri->base; void __iomem *addr; u32 bar0, bar1; u16 cmd; @@ -395,4 +533,12 @@ resource_size_t cxl_rcrb_to_component(struct device *dev, return component_reg_phys; } -EXPORT_SYMBOL_NS_GPL(cxl_rcrb_to_component, CXL); + +resource_size_t cxl_rcd_component_reg_phys(struct device *dev, + struct cxl_dport *dport) +{ + if (!dport->rch) + return CXL_RESOURCE_NONE; + return __rcrb_to_component(dev, &dport->rcrb, CXL_RCRB_UPSTREAM); +} +EXPORT_SYMBOL_NS_GPL(cxl_rcd_component_reg_phys, CXL); |