diff options
Diffstat (limited to 'drivers/vdpa')
-rw-r--r-- | drivers/vdpa/Kconfig | 10 | ||||
-rw-r--r-- | drivers/vdpa/Makefile | 1 | ||||
-rw-r--r-- | drivers/vdpa/ifcvf/ifcvf_base.c | 201 | ||||
-rw-r--r-- | drivers/vdpa/ifcvf/ifcvf_base.h | 48 | ||||
-rw-r--r-- | drivers/vdpa/ifcvf/ifcvf_main.c | 108 | ||||
-rw-r--r-- | drivers/vdpa/mlx5/net/mlx5_vnet.c | 165 | ||||
-rw-r--r-- | drivers/vdpa/mlx5/net/mlx5_vnet.h | 15 | ||||
-rw-r--r-- | drivers/vdpa/pds/Makefile | 10 | ||||
-rw-r--r-- | drivers/vdpa/pds/aux_drv.c | 140 | ||||
-rw-r--r-- | drivers/vdpa/pds/aux_drv.h | 26 | ||||
-rw-r--r-- | drivers/vdpa/pds/cmds.c | 185 | ||||
-rw-r--r-- | drivers/vdpa/pds/cmds.h | 18 | ||||
-rw-r--r-- | drivers/vdpa/pds/debugfs.c | 289 | ||||
-rw-r--r-- | drivers/vdpa/pds/debugfs.h | 17 | ||||
-rw-r--r-- | drivers/vdpa/pds/vdpa_dev.c | 769 | ||||
-rw-r--r-- | drivers/vdpa/pds/vdpa_dev.h | 49 | ||||
-rw-r--r-- | drivers/vdpa/solidrun/snet_ctrl.c | 6 | ||||
-rw-r--r-- | drivers/vdpa/solidrun/snet_hwmon.c | 2 | ||||
-rw-r--r-- | drivers/vdpa/solidrun/snet_main.c | 15 | ||||
-rw-r--r-- | drivers/vdpa/solidrun/snet_vdpa.h | 1 | ||||
-rw-r--r-- | drivers/vdpa/vdpa_user/vduse_dev.c | 6 |
21 files changed, 1880 insertions, 201 deletions
diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig index cd6ad92f3f05..656c1cb541de 100644 --- a/drivers/vdpa/Kconfig +++ b/drivers/vdpa/Kconfig @@ -116,4 +116,14 @@ config ALIBABA_ENI_VDPA This driver includes a HW monitor device that reads health values from the DPU. +config PDS_VDPA + tristate "vDPA driver for AMD/Pensando DSC devices" + select VIRTIO_PCI_LIB + depends on PCI_MSI + depends on PDS_CORE + help + vDPA network driver for AMD/Pensando's PDS Core devices. + With this driver, the VirtIO dataplane can be + offloaded to an AMD/Pensando DSC device. + endif # VDPA diff --git a/drivers/vdpa/Makefile b/drivers/vdpa/Makefile index 59396ff2a318..8f53c6f3cca7 100644 --- a/drivers/vdpa/Makefile +++ b/drivers/vdpa/Makefile @@ -7,3 +7,4 @@ obj-$(CONFIG_MLX5_VDPA) += mlx5/ obj-$(CONFIG_VP_VDPA) += virtio_pci/ obj-$(CONFIG_ALIBABA_ENI_VDPA) += alibaba/ obj-$(CONFIG_SNET_VDPA) += solidrun/ +obj-$(CONFIG_PDS_VDPA) += pds/ diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c index 5563b3a773c7..060f837a4f9f 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.c +++ b/drivers/vdpa/ifcvf/ifcvf_base.c @@ -69,6 +69,37 @@ static int ifcvf_read_config_range(struct pci_dev *dev, return 0; } +static u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid) +{ + u16 queue_size; + + vp_iowrite16(qid, &hw->common_cfg->queue_select); + queue_size = vp_ioread16(&hw->common_cfg->queue_size); + + return queue_size; +} + +/* This function returns the max allowed safe size for + * all virtqueues. It is the minimal size that can be + * suppprted by all virtqueues. + */ +u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw) +{ + u16 queue_size, max_size, qid; + + max_size = ifcvf_get_vq_size(hw, 0); + for (qid = 1; qid < hw->nr_vring; qid++) { + queue_size = ifcvf_get_vq_size(hw, qid); + /* 0 means the queue is unavailable */ + if (!queue_size) + continue; + + max_size = min(queue_size, max_size); + } + + return max_size; +} + int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev) { struct virtio_pci_cap cap; @@ -134,6 +165,9 @@ next: } hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues); + hw->vring = kzalloc(sizeof(struct vring_info) * hw->nr_vring, GFP_KERNEL); + if (!hw->vring) + return -ENOMEM; for (i = 0; i < hw->nr_vring; i++) { vp_iowrite16(i, &hw->common_cfg->queue_select); @@ -170,21 +204,9 @@ void ifcvf_set_status(struct ifcvf_hw *hw, u8 status) void ifcvf_reset(struct ifcvf_hw *hw) { - hw->config_cb.callback = NULL; - hw->config_cb.private = NULL; - ifcvf_set_status(hw, 0); - /* flush set_status, make sure VF is stopped, reset */ - ifcvf_get_status(hw); -} - -static void ifcvf_add_status(struct ifcvf_hw *hw, u8 status) -{ - if (status != 0) - status |= ifcvf_get_status(hw); - - ifcvf_set_status(hw, status); - ifcvf_get_status(hw); + while (ifcvf_get_status(hw)) + msleep(1); } u64 ifcvf_get_hw_features(struct ifcvf_hw *hw) @@ -204,11 +226,29 @@ u64 ifcvf_get_hw_features(struct ifcvf_hw *hw) return features; } -u64 ifcvf_get_features(struct ifcvf_hw *hw) +/* return provisioned vDPA dev features */ +u64 ifcvf_get_dev_features(struct ifcvf_hw *hw) { return hw->dev_features; } +u64 ifcvf_get_driver_features(struct ifcvf_hw *hw) +{ + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; + u32 features_lo, features_hi; + u64 features; + + vp_iowrite32(0, &cfg->device_feature_select); + features_lo = vp_ioread32(&cfg->guest_feature); + + vp_iowrite32(1, &cfg->device_feature_select); + features_hi = vp_ioread32(&cfg->guest_feature); + + features = ((u64)features_hi << 32) | features_lo; + + return features; +} + int ifcvf_verify_min_features(struct ifcvf_hw *hw, u64 features) { if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) { @@ -275,7 +315,7 @@ void ifcvf_write_dev_config(struct ifcvf_hw *hw, u64 offset, vp_iowrite8(*p++, hw->dev_cfg + offset + i); } -static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features) +void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features) { struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; @@ -286,105 +326,104 @@ static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features) vp_iowrite32(features >> 32, &cfg->guest_feature); } -static int ifcvf_config_features(struct ifcvf_hw *hw) -{ - ifcvf_set_features(hw, hw->req_features); - ifcvf_add_status(hw, VIRTIO_CONFIG_S_FEATURES_OK); - - if (!(ifcvf_get_status(hw) & VIRTIO_CONFIG_S_FEATURES_OK)) { - IFCVF_ERR(hw->pdev, "Failed to set FEATURES_OK status\n"); - return -EIO; - } - - return 0; -} - u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid) { - struct ifcvf_lm_cfg __iomem *ifcvf_lm; - void __iomem *avail_idx_addr; + struct ifcvf_lm_cfg __iomem *lm_cfg = hw->lm_cfg; u16 last_avail_idx; - u32 q_pair_id; - ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; - q_pair_id = qid / 2; - avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; - last_avail_idx = vp_ioread16(avail_idx_addr); + last_avail_idx = vp_ioread16(&lm_cfg->vq_state_region + qid * 2); return last_avail_idx; } int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num) { - struct ifcvf_lm_cfg __iomem *ifcvf_lm; - void __iomem *avail_idx_addr; - u32 q_pair_id; + struct ifcvf_lm_cfg __iomem *lm_cfg = hw->lm_cfg; - ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; - q_pair_id = qid / 2; - avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; - hw->vring[qid].last_avail_idx = num; - vp_iowrite16(num, avail_idx_addr); + vp_iowrite16(num, &lm_cfg->vq_state_region + qid * 2); return 0; } -static int ifcvf_hw_enable(struct ifcvf_hw *hw) +void ifcvf_set_vq_num(struct ifcvf_hw *hw, u16 qid, u32 num) { - struct virtio_pci_common_cfg __iomem *cfg; - u32 i; + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; - cfg = hw->common_cfg; - for (i = 0; i < hw->nr_vring; i++) { - if (!hw->vring[i].ready) - break; + vp_iowrite16(qid, &cfg->queue_select); + vp_iowrite16(num, &cfg->queue_size); +} - vp_iowrite16(i, &cfg->queue_select); - vp_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, - &cfg->queue_desc_hi); - vp_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo, - &cfg->queue_avail_hi); - vp_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo, - &cfg->queue_used_hi); - vp_iowrite16(hw->vring[i].size, &cfg->queue_size); - ifcvf_set_vq_state(hw, i, hw->vring[i].last_avail_idx); - vp_iowrite16(1, &cfg->queue_enable); - } +int ifcvf_set_vq_address(struct ifcvf_hw *hw, u16 qid, u64 desc_area, + u64 driver_area, u64 device_area) +{ + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; + + vp_iowrite16(qid, &cfg->queue_select); + vp_iowrite64_twopart(desc_area, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); + vp_iowrite64_twopart(driver_area, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); + vp_iowrite64_twopart(device_area, &cfg->queue_used_lo, + &cfg->queue_used_hi); return 0; } -static void ifcvf_hw_disable(struct ifcvf_hw *hw) +bool ifcvf_get_vq_ready(struct ifcvf_hw *hw, u16 qid) { - u32 i; + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; + u16 queue_enable; - ifcvf_set_config_vector(hw, VIRTIO_MSI_NO_VECTOR); - for (i = 0; i < hw->nr_vring; i++) { - ifcvf_set_vq_vector(hw, i, VIRTIO_MSI_NO_VECTOR); - } + vp_iowrite16(qid, &cfg->queue_select); + queue_enable = vp_ioread16(&cfg->queue_enable); + + return (bool)queue_enable; } -int ifcvf_start_hw(struct ifcvf_hw *hw) +void ifcvf_set_vq_ready(struct ifcvf_hw *hw, u16 qid, bool ready) { - ifcvf_reset(hw); - ifcvf_add_status(hw, VIRTIO_CONFIG_S_ACKNOWLEDGE); - ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER); + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; - if (ifcvf_config_features(hw) < 0) - return -EINVAL; + vp_iowrite16(qid, &cfg->queue_select); + vp_iowrite16(ready, &cfg->queue_enable); +} - if (ifcvf_hw_enable(hw) < 0) - return -EINVAL; +static void ifcvf_reset_vring(struct ifcvf_hw *hw) +{ + u16 qid; - ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER_OK); + for (qid = 0; qid < hw->nr_vring; qid++) { + hw->vring[qid].cb.callback = NULL; + hw->vring[qid].cb.private = NULL; + ifcvf_set_vq_vector(hw, qid, VIRTIO_MSI_NO_VECTOR); + } +} - return 0; +static void ifcvf_reset_config_handler(struct ifcvf_hw *hw) +{ + hw->config_cb.callback = NULL; + hw->config_cb.private = NULL; + ifcvf_set_config_vector(hw, VIRTIO_MSI_NO_VECTOR); +} + +static void ifcvf_synchronize_irq(struct ifcvf_hw *hw) +{ + u32 nvectors = hw->num_msix_vectors; + struct pci_dev *pdev = hw->pdev; + int i, irq; + + for (i = 0; i < nvectors; i++) { + irq = pci_irq_vector(pdev, i); + if (irq >= 0) + synchronize_irq(irq); + } } -void ifcvf_stop_hw(struct ifcvf_hw *hw) +void ifcvf_stop(struct ifcvf_hw *hw) { - ifcvf_hw_disable(hw); - ifcvf_reset(hw); + ifcvf_synchronize_irq(hw); + ifcvf_reset_vring(hw); + ifcvf_reset_config_handler(hw); } void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid) diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h index c20d1c40214e..b57849c643f6 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.h +++ b/drivers/vdpa/ifcvf/ifcvf_base.h @@ -24,15 +24,9 @@ #define N3000_DEVICE_ID 0x1041 #define N3000_SUBSYS_DEVICE_ID 0x001A -/* Max 8 data queue pairs(16 queues) and one control vq for now. */ -#define IFCVF_MAX_QUEUES 17 - #define IFCVF_QUEUE_ALIGNMENT PAGE_SIZE -#define IFCVF_QUEUE_MAX 32768 #define IFCVF_PCI_MAX_RESOURCE 6 -#define IFCVF_LM_CFG_SIZE 0x40 -#define IFCVF_LM_RING_STATE_OFFSET 0x20 #define IFCVF_LM_BAR 4 #define IFCVF_ERR(pdev, fmt, ...) dev_err(&pdev->dev, fmt, ##__VA_ARGS__) @@ -47,12 +41,7 @@ #define MSIX_VECTOR_DEV_SHARED 3 struct vring_info { - u64 desc; - u64 avail; - u64 used; - u16 size; u16 last_avail_idx; - bool ready; void __iomem *notify_addr; phys_addr_t notify_pa; u32 irq; @@ -60,10 +49,18 @@ struct vring_info { char msix_name[256]; }; +struct ifcvf_lm_cfg { + __le64 control; + __le64 status; + __le64 lm_mem_log_start_addr; + __le64 lm_mem_log_end_addr; + __le16 vq_state_region; +}; + struct ifcvf_hw { u8 __iomem *isr; /* Live migration */ - u8 __iomem *lm_cfg; + struct ifcvf_lm_cfg __iomem *lm_cfg; /* Notification bar number */ u8 notify_bar; u8 msix_vector_status; @@ -74,13 +71,12 @@ struct ifcvf_hw { phys_addr_t notify_base_pa; u32 notify_off_multiplier; u32 dev_type; - u64 req_features; u64 hw_features; /* provisioned device features */ u64 dev_features; struct virtio_pci_common_cfg __iomem *common_cfg; void __iomem *dev_cfg; - struct vring_info vring[IFCVF_MAX_QUEUES]; + struct vring_info *vring; void __iomem * const *base; char config_msix_name[256]; struct vdpa_callback config_cb; @@ -88,6 +84,7 @@ struct ifcvf_hw { int vqs_reused_irq; u16 nr_vring; /* VIRTIO_PCI_CAP_DEVICE_CFG size */ + u32 num_msix_vectors; u32 cap_dev_config_size; struct pci_dev *pdev; }; @@ -98,16 +95,6 @@ struct ifcvf_adapter { struct ifcvf_hw *vf; }; -struct ifcvf_vring_lm_cfg { - u32 idx_addr[2]; - u8 reserved[IFCVF_LM_CFG_SIZE - 8]; -}; - -struct ifcvf_lm_cfg { - u8 reserved[IFCVF_LM_RING_STATE_OFFSET]; - struct ifcvf_vring_lm_cfg vring_lm_cfg[IFCVF_MAX_QUEUES]; -}; - struct ifcvf_vdpa_mgmt_dev { struct vdpa_mgmt_dev mdev; struct ifcvf_hw vf; @@ -116,8 +103,7 @@ struct ifcvf_vdpa_mgmt_dev { }; int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev); -int ifcvf_start_hw(struct ifcvf_hw *hw); -void ifcvf_stop_hw(struct ifcvf_hw *hw); +void ifcvf_stop(struct ifcvf_hw *hw); void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid); void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset, void *dst, int length); @@ -127,7 +113,7 @@ u8 ifcvf_get_status(struct ifcvf_hw *hw); void ifcvf_set_status(struct ifcvf_hw *hw, u8 status); void io_write64_twopart(u64 val, u32 *lo, u32 *hi); void ifcvf_reset(struct ifcvf_hw *hw); -u64 ifcvf_get_features(struct ifcvf_hw *hw); +u64 ifcvf_get_dev_features(struct ifcvf_hw *hw); u64 ifcvf_get_hw_features(struct ifcvf_hw *hw); int ifcvf_verify_min_features(struct ifcvf_hw *hw, u64 features); u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid); @@ -137,4 +123,12 @@ int ifcvf_probed_virtio_net(struct ifcvf_hw *hw); u32 ifcvf_get_config_size(struct ifcvf_hw *hw); u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector); u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector); +void ifcvf_set_vq_num(struct ifcvf_hw *hw, u16 qid, u32 num); +int ifcvf_set_vq_address(struct ifcvf_hw *hw, u16 qid, u64 desc_area, + u64 driver_area, u64 device_area); +bool ifcvf_get_vq_ready(struct ifcvf_hw *hw, u16 qid); +void ifcvf_set_vq_ready(struct ifcvf_hw *hw, u16 qid, bool ready); +void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features); +u64 ifcvf_get_driver_features(struct ifcvf_hw *hw); +u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw); #endif /* _IFCVF_H_ */ diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index 7f78c47e40d6..e98fa8100f3c 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -125,6 +125,7 @@ static void ifcvf_free_irq(struct ifcvf_hw *vf) ifcvf_free_vq_irq(vf); ifcvf_free_config_irq(vf); ifcvf_free_irq_vectors(pdev); + vf->num_msix_vectors = 0; } /* ifcvf MSIX vectors allocator, this helper tries to allocate @@ -343,56 +344,11 @@ static int ifcvf_request_irq(struct ifcvf_hw *vf) if (ret) return ret; - return 0; -} - -static int ifcvf_start_datapath(struct ifcvf_adapter *adapter) -{ - struct ifcvf_hw *vf = adapter->vf; - u8 status; - int ret; - - ret = ifcvf_start_hw(vf); - if (ret < 0) { - status = ifcvf_get_status(vf); - status |= VIRTIO_CONFIG_S_FAILED; - ifcvf_set_status(vf, status); - } - - return ret; -} - -static int ifcvf_stop_datapath(struct ifcvf_adapter *adapter) -{ - struct ifcvf_hw *vf = adapter->vf; - int i; - - for (i = 0; i < vf->nr_vring; i++) - vf->vring[i].cb.callback = NULL; - - ifcvf_stop_hw(vf); + vf->num_msix_vectors = nvectors; return 0; } -static void ifcvf_reset_vring(struct ifcvf_adapter *adapter) -{ - struct ifcvf_hw *vf = adapter->vf; - int i; - - for (i = 0; i < vf->nr_vring; i++) { - vf->vring[i].last_avail_idx = 0; - vf->vring[i].desc = 0; - vf->vring[i].avail = 0; - vf->vring[i].used = 0; - vf->vring[i].ready = 0; - vf->vring[i].cb.callback = NULL; - vf->vring[i].cb.private = NULL; - } - - ifcvf_reset(vf); -} - static struct ifcvf_adapter *vdpa_to_adapter(struct vdpa_device *vdpa_dev) { return container_of(vdpa_dev, struct ifcvf_adapter, vdpa); @@ -414,7 +370,7 @@ static u64 ifcvf_vdpa_get_device_features(struct vdpa_device *vdpa_dev) u64 features; if (type == VIRTIO_ID_NET || type == VIRTIO_ID_BLOCK) - features = ifcvf_get_features(vf); + features = ifcvf_get_dev_features(vf); else { features = 0; IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", vf->dev_type); @@ -432,7 +388,7 @@ static int ifcvf_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 feat if (ret) return ret; - vf->req_features = features; + ifcvf_set_driver_features(vf, features); return 0; } @@ -440,8 +396,11 @@ static int ifcvf_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 feat static u64 ifcvf_vdpa_get_driver_features(struct vdpa_device *vdpa_dev) { struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + u64 features; + + features = ifcvf_get_driver_features(vf); - return vf->req_features; + return features; } static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev) @@ -453,13 +412,11 @@ static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev) static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) { - struct ifcvf_adapter *adapter; struct ifcvf_hw *vf; u8 status_old; int ret; vf = vdpa_to_vf(vdpa_dev); - adapter = vdpa_to_adapter(vdpa_dev); status_old = ifcvf_get_status(vf); if (status_old == status) @@ -469,16 +426,9 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) { ret = ifcvf_request_irq(vf); if (ret) { - status = ifcvf_get_status(vf); - status |= VIRTIO_CONFIG_S_FAILED; - ifcvf_set_status(vf, status); + IFCVF_ERR(vf->pdev, "failed to request irq with error %d\n", ret); return; } - - if (ifcvf_start_datapath(adapter) < 0) - IFCVF_ERR(adapter->pdev, - "Failed to set ifcvf vdpa status %u\n", - status); } ifcvf_set_status(vf, status); @@ -486,30 +436,24 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev) { - struct ifcvf_adapter *adapter; - struct ifcvf_hw *vf; - u8 status_old; - - vf = vdpa_to_vf(vdpa_dev); - adapter = vdpa_to_adapter(vdpa_dev); - status_old = ifcvf_get_status(vf); + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + u8 status = ifcvf_get_status(vf); - if (status_old == 0) - return 0; + ifcvf_stop(vf); - if (status_old & VIRTIO_CONFIG_S_DRIVER_OK) { - ifcvf_stop_datapath(adapter); + if (status & VIRTIO_CONFIG_S_DRIVER_OK) ifcvf_free_irq(vf); - } - ifcvf_reset_vring(adapter); + ifcvf_reset(vf); return 0; } static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev) { - return IFCVF_QUEUE_MAX; + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + return ifcvf_get_max_vq_size(vf); } static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid, @@ -542,14 +486,14 @@ static void ifcvf_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, { struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); - vf->vring[qid].ready = ready; + ifcvf_set_vq_ready(vf, qid, ready); } static bool ifcvf_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid) { struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); - return vf->vring[qid].ready; + return ifcvf_get_vq_ready(vf, qid); } static void ifcvf_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, @@ -557,7 +501,7 @@ static void ifcvf_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, { struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); - vf->vring[qid].size = num; + ifcvf_set_vq_num(vf, qid, num); } static int ifcvf_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, @@ -566,11 +510,7 @@ static int ifcvf_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, { struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); - vf->vring[qid].desc = desc_area; - vf->vring[qid].avail = driver_area; - vf->vring[qid].used = device_area; - - return 0; + return ifcvf_set_vq_address(vf, qid, desc_area, driver_area, device_area); } static void ifcvf_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid) @@ -892,6 +832,7 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; err: + kfree(ifcvf_mgmt_dev->vf.vring); kfree(ifcvf_mgmt_dev); return ret; } @@ -902,6 +843,7 @@ static void ifcvf_remove(struct pci_dev *pdev) ifcvf_mgmt_dev = pci_get_drvdata(pdev); vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev); + kfree(ifcvf_mgmt_dev->vf.vring); kfree(ifcvf_mgmt_dev); } @@ -911,7 +853,9 @@ static struct pci_device_id ifcvf_pci_ids[] = { N3000_DEVICE_ID, PCI_VENDOR_ID_INTEL, N3000_SUBSYS_DEVICE_ID) }, - /* C5000X-PL network device */ + /* C5000X-PL network device + * F2000X-PL network device + */ { PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET, VIRTIO_TRANS_ID_NET, PCI_VENDOR_ID_INTEL, diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index 279ac6a558d2..9138ef2fb2c8 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -83,6 +83,7 @@ struct mlx5_vq_restore_info { u64 driver_addr; u16 avail_index; u16 used_index; + struct msi_map map; bool ready; bool restore; }; @@ -118,6 +119,7 @@ struct mlx5_vdpa_virtqueue { u16 avail_idx; u16 used_idx; int fw_state; + struct msi_map map; /* keep last in the struct */ struct mlx5_vq_restore_info ri; @@ -808,6 +810,13 @@ static bool counters_supported(const struct mlx5_vdpa_dev *mvdev) BIT_ULL(MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); } +static bool msix_mode_supported(struct mlx5_vdpa_dev *mvdev) +{ + return MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, event_mode) & + (1 << MLX5_VIRTIO_Q_EVENT_MODE_MSIX_MODE) && + pci_msix_can_alloc_dyn(mvdev->mdev->pdev); +} + static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); @@ -849,9 +858,15 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque if (vq_is_tx(mvq->index)) MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); - MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); + if (mvq->map.virq) { + MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_MSIX_MODE); + MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->map.index); + } else { + MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); + MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); + } + MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); - MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); @@ -1194,6 +1209,56 @@ static void counter_set_dealloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_vir mlx5_vdpa_warn(&ndev->mvdev, "dealloc counter set 0x%x\n", mvq->counter_set_id); } +static irqreturn_t mlx5_vdpa_int_handler(int irq, void *priv) +{ + struct vdpa_callback *cb = priv; + + if (cb->callback) + return cb->callback(cb->private); + + return IRQ_HANDLED; +} + +static void alloc_vector(struct mlx5_vdpa_net *ndev, + struct mlx5_vdpa_virtqueue *mvq) +{ + struct mlx5_vdpa_irq_pool *irqp = &ndev->irqp; + struct mlx5_vdpa_irq_pool_entry *ent; + int err; + int i; + + for (i = 0; i < irqp->num_ent; i++) { + ent = &irqp->entries[i]; + if (!ent->used) { + snprintf(ent->name, MLX5_VDPA_IRQ_NAME_LEN, "%s-vq-%d", + dev_name(&ndev->mvdev.vdev.dev), mvq->index); + ent->dev_id = &ndev->event_cbs[mvq->index]; + err = request_irq(ent->map.virq, mlx5_vdpa_int_handler, 0, + ent->name, ent->dev_id); + if (err) + return; + + ent->used = true; + mvq->map = ent->map; + return; + } + } +} + +static void dealloc_vector(struct mlx5_vdpa_net *ndev, + struct mlx5_vdpa_virtqueue *mvq) +{ + struct mlx5_vdpa_irq_pool *irqp = &ndev->irqp; + int i; + + for (i = 0; i < irqp->num_ent; i++) + if (mvq->map.virq == irqp->entries[i].map.virq) { + free_irq(mvq->map.virq, irqp->entries[i].dev_id); + irqp->entries[i].used = false; + return; + } +} + static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { u16 idx = mvq->index; @@ -1223,27 +1288,31 @@ static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) err = counter_set_alloc(ndev, mvq); if (err) - goto err_counter; + goto err_connect; + alloc_vector(ndev, mvq); err = create_virtqueue(ndev, mvq); if (err) - goto err_connect; + goto err_vq; if (mvq->ready) { err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); if (err) { mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", idx, err); - goto err_connect; + goto err_modify; } } mvq->initialized = true; return 0; -err_connect: +err_modify: + destroy_virtqueue(ndev, mvq); +err_vq: + dealloc_vector(ndev, mvq); counter_set_dealloc(ndev, mvq); -err_counter: +err_connect: qp_destroy(ndev, &mvq->vqqp); err_vqqp: qp_destroy(ndev, &mvq->fwqp); @@ -1288,6 +1357,7 @@ static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue * suspend_vq(ndev, mvq); destroy_virtqueue(ndev, mvq); + dealloc_vector(ndev, mvq); counter_set_dealloc(ndev, mvq); qp_destroy(ndev, &mvq->vqqp); qp_destroy(ndev, &mvq->fwqp); @@ -2505,6 +2575,7 @@ static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqu ri->desc_addr = mvq->desc_addr; ri->device_addr = mvq->device_addr; ri->driver_addr = mvq->driver_addr; + ri->map = mvq->map; ri->restore = true; return 0; } @@ -2549,6 +2620,7 @@ static void restore_channels_info(struct mlx5_vdpa_net *ndev) mvq->desc_addr = ri->desc_addr; mvq->device_addr = ri->device_addr; mvq->driver_addr = ri->driver_addr; + mvq->map = ri->map; } } @@ -2833,6 +2905,25 @@ static struct device *mlx5_get_vq_dma_dev(struct vdpa_device *vdev, u16 idx) return mvdev->vdev.dma_dev; } +static void free_irqs(struct mlx5_vdpa_net *ndev) +{ + struct mlx5_vdpa_irq_pool_entry *ent; + int i; + + if (!msix_mode_supported(&ndev->mvdev)) + return; + + if (!ndev->irqp.entries) + return; + + for (i = ndev->irqp.num_ent - 1; i >= 0; i--) { + ent = ndev->irqp.entries + i; + if (ent->map.virq) + pci_msix_free_irq(ndev->mvdev.mdev->pdev, ent->map); + } + kfree(ndev->irqp.entries); +} + static void mlx5_vdpa_free(struct vdpa_device *vdev) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); @@ -2848,6 +2939,7 @@ static void mlx5_vdpa_free(struct vdpa_device *vdev) mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); } mlx5_vdpa_free_resources(&ndev->mvdev); + free_irqs(ndev); kfree(ndev->event_cbs); kfree(ndev->vqs); } @@ -2876,9 +2968,23 @@ static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device return ret; } -static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) +static int mlx5_get_vq_irq(struct vdpa_device *vdev, u16 idx) { - return -EOPNOTSUPP; + struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); + struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); + struct mlx5_vdpa_virtqueue *mvq; + + if (!is_index_valid(mvdev, idx)) + return -EINVAL; + + if (is_ctrl_vq_idx(mvdev, idx)) + return -EOPNOTSUPP; + + mvq = &ndev->vqs[idx]; + if (!mvq->map.virq) + return -EOPNOTSUPP; + + return mvq->map.virq; } static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) @@ -3155,6 +3261,34 @@ static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu) return err; } +static void allocate_irqs(struct mlx5_vdpa_net *ndev) +{ + struct mlx5_vdpa_irq_pool_entry *ent; + int i; + + if (!msix_mode_supported(&ndev->mvdev)) + return; + + if (!ndev->mvdev.mdev->pdev) + return; + + ndev->irqp.entries = kcalloc(ndev->mvdev.max_vqs, sizeof(*ndev->irqp.entries), GFP_KERNEL); + if (!ndev->irqp.entries) + return; + + + for (i = 0; i < ndev->mvdev.max_vqs; i++) { + ent = ndev->irqp.entries + i; + snprintf(ent->name, MLX5_VDPA_IRQ_NAME_LEN, "%s-vq-%d", + dev_name(&ndev->mvdev.vdev.dev), i); + ent->map = pci_msix_alloc_irq_at(ndev->mvdev.mdev->pdev, MSI_ANY_INDEX, NULL); + if (!ent->map.virq) + return; + + ndev->irqp.num_ent++; + } +} + static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, const struct vdpa_dev_set_config *add_config) { @@ -3233,6 +3367,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, } init_mvqs(ndev); + allocate_irqs(ndev); init_rwsem(&ndev->reslock); config = &ndev->config; @@ -3413,6 +3548,17 @@ static void mlx5v_remove(struct auxiliary_device *adev) kfree(mgtdev); } +static void mlx5v_shutdown(struct auxiliary_device *auxdev) +{ + struct mlx5_vdpa_mgmtdev *mgtdev; + struct mlx5_vdpa_net *ndev; + + mgtdev = auxiliary_get_drvdata(auxdev); + ndev = mgtdev->ndev; + + free_irqs(ndev); +} + static const struct auxiliary_device_id mlx5v_id_table[] = { { .name = MLX5_ADEV_NAME ".vnet", }, {}, @@ -3424,6 +3570,7 @@ static struct auxiliary_driver mlx5v_driver = { .name = "vnet", .probe = mlx5v_probe, .remove = mlx5v_remove, + .shutdown = mlx5v_shutdown, .id_table = mlx5v_id_table, }; diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.h b/drivers/vdpa/mlx5/net/mlx5_vnet.h index c90a89e1de4d..36c44d9fdd16 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.h +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.h @@ -26,6 +26,20 @@ static inline u16 key2vid(u64 key) return (u16)(key >> 48) & 0xfff; } +#define MLX5_VDPA_IRQ_NAME_LEN 32 + +struct mlx5_vdpa_irq_pool_entry { + struct msi_map map; + bool used; + char name[MLX5_VDPA_IRQ_NAME_LEN]; + void *dev_id; +}; + +struct mlx5_vdpa_irq_pool { + int num_ent; + struct mlx5_vdpa_irq_pool_entry *entries; +}; + struct mlx5_vdpa_net { struct mlx5_vdpa_dev mvdev; struct mlx5_vdpa_net_resources res; @@ -49,6 +63,7 @@ struct mlx5_vdpa_net { struct vdpa_callback config_cb; struct mlx5_vdpa_wq_ent cvq_ent; struct hlist_head macvlan_hash[MLX5V_MACVLAN_SIZE]; + struct mlx5_vdpa_irq_pool irqp; struct dentry *debugfs; }; diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile new file mode 100644 index 000000000000..2e22418e3ab3 --- /dev/null +++ b/drivers/vdpa/pds/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright(c) 2023 Advanced Micro Devices, Inc + +obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o + +pds_vdpa-y := aux_drv.o \ + cmds.o \ + vdpa_dev.o + +pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c new file mode 100644 index 000000000000..186e9ee22eb1 --- /dev/null +++ b/drivers/vdpa/pds/aux_drv.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ + +#include <linux/auxiliary_bus.h> +#include <linux/pci.h> +#include <linux/vdpa.h> +#include <linux/virtio_pci_modern.h> + +#include <linux/pds/pds_common.h> +#include <linux/pds/pds_core_if.h> +#include <linux/pds/pds_adminq.h> +#include <linux/pds/pds_auxbus.h> + +#include "aux_drv.h" +#include "debugfs.h" +#include "vdpa_dev.h" + +static const struct auxiliary_device_id pds_vdpa_id_table[] = { + { .name = PDS_VDPA_DEV_NAME, }, + {}, +}; + +static int pds_vdpa_device_id_check(struct pci_dev *pdev) +{ + if (pdev->device != PCI_DEVICE_ID_PENSANDO_VDPA_VF || + pdev->vendor != PCI_VENDOR_ID_PENSANDO) + return -ENODEV; + + return PCI_DEVICE_ID_PENSANDO_VDPA_VF; +} + +static int pds_vdpa_probe(struct auxiliary_device *aux_dev, + const struct auxiliary_device_id *id) + +{ + struct pds_auxiliary_dev *padev = + container_of(aux_dev, struct pds_auxiliary_dev, aux_dev); + struct device *dev = &aux_dev->dev; + struct pds_vdpa_aux *vdpa_aux; + int err; + + vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL); + if (!vdpa_aux) + return -ENOMEM; + + vdpa_aux->padev = padev; + vdpa_aux->vf_id = pci_iov_vf_id(padev->vf_pdev); + auxiliary_set_drvdata(aux_dev, vdpa_aux); + + /* Get device ident info and set up the vdpa_mgmt_dev */ + err = pds_vdpa_get_mgmt_info(vdpa_aux); + if (err) + goto err_free_mem; + + /* Find the virtio configuration */ + vdpa_aux->vd_mdev.pci_dev = padev->vf_pdev; + vdpa_aux->vd_mdev.device_id_check = pds_vdpa_device_id_check; + vdpa_aux->vd_mdev.dma_mask = DMA_BIT_MASK(PDS_CORE_ADDR_LEN); + err = vp_modern_probe(&vdpa_aux->vd_mdev); + if (err) { + dev_err(dev, "Unable to probe for virtio configuration: %pe\n", + ERR_PTR(err)); + goto err_free_mgmt_info; + } + + /* Let vdpa know that we can provide devices */ + err = vdpa_mgmtdev_register(&vdpa_aux->vdpa_mdev); + if (err) { + dev_err(dev, "%s: Failed to initialize vdpa_mgmt interface: %pe\n", + __func__, ERR_PTR(err)); + goto err_free_virtio; + } + + pds_vdpa_debugfs_add_pcidev(vdpa_aux); + pds_vdpa_debugfs_add_ident(vdpa_aux); + + return 0; + +err_free_virtio: + vp_modern_remove(&vdpa_aux->vd_mdev); +err_free_mgmt_info: + pci_free_irq_vectors(padev->vf_pdev); +err_free_mem: + kfree(vdpa_aux); + auxiliary_set_drvdata(aux_dev, NULL); + + return err; +} + +static void pds_vdpa_remove(struct auxiliary_device *aux_dev) +{ + struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev); + struct device *dev = &aux_dev->dev; + + vdpa_mgmtdev_unregister(&vdpa_aux->vdpa_mdev); + vp_modern_remove(&vdpa_aux->vd_mdev); + pci_free_irq_vectors(vdpa_aux->padev->vf_pdev); + + pds_vdpa_debugfs_del_vdpadev(vdpa_aux); + kfree(vdpa_aux); + auxiliary_set_drvdata(aux_dev, NULL); + + dev_info(dev, "Removed\n"); +} + +static struct auxiliary_driver pds_vdpa_driver = { + .name = PDS_DEV_TYPE_VDPA_STR, + .probe = pds_vdpa_probe, + .remove = pds_vdpa_remove, + .id_table = pds_vdpa_id_table, +}; + +static void __exit pds_vdpa_cleanup(void) +{ + auxiliary_driver_unregister(&pds_vdpa_driver); + + pds_vdpa_debugfs_destroy(); +} +module_exit(pds_vdpa_cleanup); + +static int __init pds_vdpa_init(void) +{ + int err; + + pds_vdpa_debugfs_create(); + + err = auxiliary_driver_register(&pds_vdpa_driver); + if (err) { + pr_err("%s: aux driver register failed: %pe\n", + PDS_VDPA_DRV_NAME, ERR_PTR(err)); + pds_vdpa_debugfs_destroy(); + } + + return err; +} +module_init(pds_vdpa_init); + +MODULE_DESCRIPTION(PDS_VDPA_DRV_DESCRIPTION); +MODULE_AUTHOR("Advanced Micro Devices, Inc"); +MODULE_LICENSE("GPL"); diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h new file mode 100644 index 000000000000..26b75344156e --- /dev/null +++ b/drivers/vdpa/pds/aux_drv.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ + +#ifndef _AUX_DRV_H_ +#define _AUX_DRV_H_ + +#include <linux/virtio_pci_modern.h> + +#define PDS_VDPA_DRV_DESCRIPTION "AMD/Pensando vDPA VF Device Driver" +#define PDS_VDPA_DRV_NAME KBUILD_MODNAME + +struct pds_vdpa_aux { + struct pds_auxiliary_dev *padev; + + struct vdpa_mgmt_dev vdpa_mdev; + struct pds_vdpa_device *pdsv; + + struct pds_vdpa_ident ident; + + int vf_id; + struct dentry *dentry; + struct virtio_pci_modern_device vd_mdev; + + int nintrs; +}; +#endif /* _AUX_DRV_H_ */ diff --git a/drivers/vdpa/pds/cmds.c b/drivers/vdpa/pds/cmds.c new file mode 100644 index 000000000000..80863a41c3cd --- /dev/null +++ b/drivers/vdpa/pds/cmds.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ + +#include <linux/vdpa.h> +#include <linux/virtio_pci_modern.h> + +#include <linux/pds/pds_common.h> +#include <linux/pds/pds_core_if.h> +#include <linux/pds/pds_adminq.h> +#include <linux/pds/pds_auxbus.h> + +#include "vdpa_dev.h" +#include "aux_drv.h" +#include "cmds.h" + +int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv) +{ + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; + struct device *dev = &padev->aux_dev.dev; + union pds_core_adminq_cmd cmd = { + .vdpa_init.opcode = PDS_VDPA_CMD_INIT, + .vdpa_init.vdpa_index = pdsv->vdpa_index, + .vdpa_init.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), + }; + union pds_core_adminq_comp comp = {}; + int err; + + /* Initialize the vdpa/virtio device */ + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_init), + &comp, 0); + if (err) + dev_dbg(dev, "Failed to init hw, status %d: %pe\n", + comp.status, ERR_PTR(err)); + + return err; +} + +int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv) +{ + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; + struct device *dev = &padev->aux_dev.dev; + union pds_core_adminq_cmd cmd = { + .vdpa.opcode = PDS_VDPA_CMD_RESET, + .vdpa.vdpa_index = pdsv->vdpa_index, + .vdpa.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), + }; + union pds_core_adminq_comp comp = {}; + int err; + + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa), &comp, 0); + if (err) + dev_dbg(dev, "Failed to reset hw, status %d: %pe\n", + comp.status, ERR_PTR(err)); + + return err; +} + +int pds_vdpa_cmd_set_status(struct pds_vdpa_device *pdsv, u8 status) +{ + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; + struct device *dev = &padev->aux_dev.dev; + union pds_core_adminq_cmd cmd = { + .vdpa_status.opcode = PDS_VDPA_CMD_STATUS_UPDATE, + .vdpa_status.vdpa_index = pdsv->vdpa_index, + .vdpa_status.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), + .vdpa_status.status = status, + }; + union pds_core_adminq_comp comp = {}; + int err; + + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_status), &comp, 0); + if (err) + dev_dbg(dev, "Failed to set status to %#x, error status %d: %pe\n", + status, comp.status, ERR_PTR(err)); + + return err; +} + +int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac) +{ + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; + struct device *dev = &padev->aux_dev.dev; + union pds_core_adminq_cmd cmd = { + .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR, + .vdpa_setattr.vdpa_index = pdsv->vdpa_index, + .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), + .vdpa_setattr.attr = PDS_VDPA_ATTR_MAC, + }; + union pds_core_adminq_comp comp = {}; + int err; + + ether_addr_copy(cmd.vdpa_setattr.mac, mac); + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_setattr), + &comp, 0); + if (err) + dev_dbg(dev, "Failed to set mac address %pM, status %d: %pe\n", + mac, comp.status, ERR_PTR(err)); + + return err; +} + +int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp) +{ + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; + struct device *dev = &padev->aux_dev.dev; + union pds_core_adminq_cmd cmd = { + .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR, + .vdpa_setattr.vdpa_index = pdsv->vdpa_index, + .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), + .vdpa_setattr.attr = PDS_VDPA_ATTR_MAX_VQ_PAIRS, + .vdpa_setattr.max_vq_pairs = cpu_to_le16(max_vqp), + }; + union pds_core_adminq_comp comp = {}; + int err; + + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_setattr), + &comp, 0); + if (err) + dev_dbg(dev, "Failed to set max vq pairs %u, status %d: %pe\n", + max_vqp, comp.status, ERR_PTR(err)); + + return err; +} + +int pds_vdpa_cmd_init_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx, + struct pds_vdpa_vq_info *vq_info) +{ + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; + struct device *dev = &padev->aux_dev.dev; + union pds_core_adminq_cmd cmd = { + .vdpa_vq_init.opcode = PDS_VDPA_CMD_VQ_INIT, + .vdpa_vq_init.vdpa_index = pdsv->vdpa_index, + .vdpa_vq_init.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), + .vdpa_vq_init.qid = cpu_to_le16(qid), + .vdpa_vq_init.len = cpu_to_le16(ilog2(vq_info->q_len)), + .vdpa_vq_init.desc_addr = cpu_to_le64(vq_info->desc_addr), + .vdpa_vq_init.avail_addr = cpu_to_le64(vq_info->avail_addr), + .vdpa_vq_init.used_addr = cpu_to_le64(vq_info->used_addr), + .vdpa_vq_init.intr_index = cpu_to_le16(qid), + .vdpa_vq_init.avail_index = cpu_to_le16(vq_info->avail_idx ^ invert_idx), + .vdpa_vq_init.used_index = cpu_to_le16(vq_info->used_idx ^ invert_idx), + }; + union pds_core_adminq_comp comp = {}; + int err; + + dev_dbg(dev, "%s: qid %d len %d desc_addr %#llx avail_addr %#llx used_addr %#llx\n", + __func__, qid, ilog2(vq_info->q_len), + vq_info->desc_addr, vq_info->avail_addr, vq_info->used_addr); + + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_vq_init), + &comp, 0); + if (err) + dev_dbg(dev, "Failed to init vq %d, status %d: %pe\n", + qid, comp.status, ERR_PTR(err)); + + return err; +} + +int pds_vdpa_cmd_reset_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx, + struct pds_vdpa_vq_info *vq_info) +{ + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; + struct device *dev = &padev->aux_dev.dev; + union pds_core_adminq_cmd cmd = { + .vdpa_vq_reset.opcode = PDS_VDPA_CMD_VQ_RESET, + .vdpa_vq_reset.vdpa_index = pdsv->vdpa_index, + .vdpa_vq_reset.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), + .vdpa_vq_reset.qid = cpu_to_le16(qid), + }; + union pds_core_adminq_comp comp = {}; + int err; + + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_vq_reset), + &comp, 0); + if (err) { + dev_dbg(dev, "Failed to reset vq %d, status %d: %pe\n", + qid, comp.status, ERR_PTR(err)); + return err; + } + + vq_info->avail_idx = le16_to_cpu(comp.vdpa_vq_reset.avail_index) ^ invert_idx; + vq_info->used_idx = le16_to_cpu(comp.vdpa_vq_reset.used_index) ^ invert_idx; + + return 0; +} diff --git a/drivers/vdpa/pds/cmds.h b/drivers/vdpa/pds/cmds.h new file mode 100644 index 000000000000..e24d85cb8f1c --- /dev/null +++ b/drivers/vdpa/pds/cmds.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ + +#ifndef _VDPA_CMDS_H_ +#define _VDPA_CMDS_H_ + +int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv); + +int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv); +int pds_vdpa_cmd_set_status(struct pds_vdpa_device *pdsv, u8 status); +int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac); +int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp); +int pds_vdpa_cmd_init_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx, + struct pds_vdpa_vq_info *vq_info); +int pds_vdpa_cmd_reset_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx, + struct pds_vdpa_vq_info *vq_info); +int pds_vdpa_cmd_set_features(struct pds_vdpa_device *pdsv, u64 features); +#endif /* _VDPA_CMDS_H_ */ diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c new file mode 100644 index 000000000000..21a0dc0cb607 --- /dev/null +++ b/drivers/vdpa/pds/debugfs.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ + +#include <linux/pci.h> +#include <linux/vdpa.h> + +#include <linux/pds/pds_common.h> +#include <linux/pds/pds_core_if.h> +#include <linux/pds/pds_adminq.h> +#include <linux/pds/pds_auxbus.h> + +#include "aux_drv.h" +#include "vdpa_dev.h" +#include "debugfs.h" + +static struct dentry *dbfs_dir; + +void pds_vdpa_debugfs_create(void) +{ + dbfs_dir = debugfs_create_dir(PDS_VDPA_DRV_NAME, NULL); +} + +void pds_vdpa_debugfs_destroy(void) +{ + debugfs_remove_recursive(dbfs_dir); + dbfs_dir = NULL; +} + +#define PRINT_SBIT_NAME(__seq, __f, __name) \ + do { \ + if ((__f) & (__name)) \ + seq_printf(__seq, " %s", &#__name[16]); \ + } while (0) + +static void print_status_bits(struct seq_file *seq, u8 status) +{ + seq_puts(seq, "status:"); + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_ACKNOWLEDGE); + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_DRIVER); + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_DRIVER_OK); + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_FEATURES_OK); + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_NEEDS_RESET); + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_FAILED); + seq_puts(seq, "\n"); +} + +static void print_feature_bits_all(struct seq_file *seq, u64 features) +{ + int i; + + seq_puts(seq, "features:"); + + for (i = 0; i < (sizeof(u64) * 8); i++) { + u64 mask = BIT_ULL(i); + + switch (features & mask) { + case BIT_ULL(VIRTIO_NET_F_CSUM): + seq_puts(seq, " VIRTIO_NET_F_CSUM"); + break; + case BIT_ULL(VIRTIO_NET_F_GUEST_CSUM): + seq_puts(seq, " VIRTIO_NET_F_GUEST_CSUM"); + break; + case BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS): + seq_puts(seq, " VIRTIO_NET_F_CTRL_GUEST_OFFLOADS"); + break; + case BIT_ULL(VIRTIO_NET_F_MTU): + seq_puts(seq, " VIRTIO_NET_F_MTU"); + break; + case BIT_ULL(VIRTIO_NET_F_MAC): + seq_puts(seq, " VIRTIO_NET_F_MAC"); + break; + case BIT_ULL(VIRTIO_NET_F_GUEST_TSO4): + seq_puts(seq, " VIRTIO_NET_F_GUEST_TSO4"); + break; + case BIT_ULL(VIRTIO_NET_F_GUEST_TSO6): + seq_puts(seq, " VIRTIO_NET_F_GUEST_TSO6"); + break; + case BIT_ULL(VIRTIO_NET_F_GUEST_ECN): + seq_puts(seq, " VIRTIO_NET_F_GUEST_ECN"); + break; + case BIT_ULL(VIRTIO_NET_F_GUEST_UFO): + seq_puts(seq, " VIRTIO_NET_F_GUEST_UFO"); + break; + case BIT_ULL(VIRTIO_NET_F_HOST_TSO4): + seq_puts(seq, " VIRTIO_NET_F_HOST_TSO4"); + break; + case BIT_ULL(VIRTIO_NET_F_HOST_TSO6): + seq_puts(seq, " VIRTIO_NET_F_HOST_TSO6"); + break; + case BIT_ULL(VIRTIO_NET_F_HOST_ECN): + seq_puts(seq, " VIRTIO_NET_F_HOST_ECN"); + break; + case BIT_ULL(VIRTIO_NET_F_HOST_UFO): + seq_puts(seq, " VIRTIO_NET_F_HOST_UFO"); + break; + case BIT_ULL(VIRTIO_NET_F_MRG_RXBUF): + seq_puts(seq, " VIRTIO_NET_F_MRG_RXBUF"); + break; + case BIT_ULL(VIRTIO_NET_F_STATUS): + seq_puts(seq, " VIRTIO_NET_F_STATUS"); + break; + case BIT_ULL(VIRTIO_NET_F_CTRL_VQ): + seq_puts(seq, " VIRTIO_NET_F_CTRL_VQ"); + break; + case BIT_ULL(VIRTIO_NET_F_CTRL_RX): + seq_puts(seq, " VIRTIO_NET_F_CTRL_RX"); + break; + case BIT_ULL(VIRTIO_NET_F_CTRL_VLAN): + seq_puts(seq, " VIRTIO_NET_F_CTRL_VLAN"); + break; + case BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA): + seq_puts(seq, " VIRTIO_NET_F_CTRL_RX_EXTRA"); + break; + case BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE): + seq_puts(seq, " VIRTIO_NET_F_GUEST_ANNOUNCE"); + break; + case BIT_ULL(VIRTIO_NET_F_MQ): + seq_puts(seq, " VIRTIO_NET_F_MQ"); + break; + case BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR): + seq_puts(seq, " VIRTIO_NET_F_CTRL_MAC_ADDR"); + break; + case BIT_ULL(VIRTIO_NET_F_HASH_REPORT): + seq_puts(seq, " VIRTIO_NET_F_HASH_REPORT"); + break; + case BIT_ULL(VIRTIO_NET_F_RSS): + seq_puts(seq, " VIRTIO_NET_F_RSS"); + break; + case BIT_ULL(VIRTIO_NET_F_RSC_EXT): + seq_puts(seq, " VIRTIO_NET_F_RSC_EXT"); + break; + case BIT_ULL(VIRTIO_NET_F_STANDBY): + seq_puts(seq, " VIRTIO_NET_F_STANDBY"); + break; + case BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX): + seq_puts(seq, " VIRTIO_NET_F_SPEED_DUPLEX"); + break; + case BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY): + seq_puts(seq, " VIRTIO_F_NOTIFY_ON_EMPTY"); + break; + case BIT_ULL(VIRTIO_F_ANY_LAYOUT): + seq_puts(seq, " VIRTIO_F_ANY_LAYOUT"); + break; + case BIT_ULL(VIRTIO_F_VERSION_1): + seq_puts(seq, " VIRTIO_F_VERSION_1"); + break; + case BIT_ULL(VIRTIO_F_ACCESS_PLATFORM): + seq_puts(seq, " VIRTIO_F_ACCESS_PLATFORM"); + break; + case BIT_ULL(VIRTIO_F_RING_PACKED): + seq_puts(seq, " VIRTIO_F_RING_PACKED"); + break; + case BIT_ULL(VIRTIO_F_ORDER_PLATFORM): + seq_puts(seq, " VIRTIO_F_ORDER_PLATFORM"); + break; + case BIT_ULL(VIRTIO_F_SR_IOV): + seq_puts(seq, " VIRTIO_F_SR_IOV"); + break; + case 0: + break; + default: + seq_printf(seq, " bit_%d", i); + break; + } + } + + seq_puts(seq, "\n"); +} + +void pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_aux *vdpa_aux) +{ + vdpa_aux->dentry = debugfs_create_dir(pci_name(vdpa_aux->padev->vf_pdev), dbfs_dir); +} + +static int identity_show(struct seq_file *seq, void *v) +{ + struct pds_vdpa_aux *vdpa_aux = seq->private; + struct vdpa_mgmt_dev *mgmt; + + seq_printf(seq, "aux_dev: %s\n", + dev_name(&vdpa_aux->padev->aux_dev.dev)); + + mgmt = &vdpa_aux->vdpa_mdev; + seq_printf(seq, "max_vqs: %d\n", mgmt->max_supported_vqs); + seq_printf(seq, "config_attr_mask: %#llx\n", mgmt->config_attr_mask); + seq_printf(seq, "supported_features: %#llx\n", mgmt->supported_features); + print_feature_bits_all(seq, mgmt->supported_features); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(identity); + +void pds_vdpa_debugfs_add_ident(struct pds_vdpa_aux *vdpa_aux) +{ + debugfs_create_file("identity", 0400, vdpa_aux->dentry, + vdpa_aux, &identity_fops); +} + +static int config_show(struct seq_file *seq, void *v) +{ + struct pds_vdpa_device *pdsv = seq->private; + struct virtio_net_config vc; + u64 driver_features; + u8 status; + + memcpy_fromio(&vc, pdsv->vdpa_aux->vd_mdev.device, + sizeof(struct virtio_net_config)); + + seq_printf(seq, "mac: %pM\n", vc.mac); + seq_printf(seq, "max_virtqueue_pairs: %d\n", + __virtio16_to_cpu(true, vc.max_virtqueue_pairs)); + seq_printf(seq, "mtu: %d\n", __virtio16_to_cpu(true, vc.mtu)); + seq_printf(seq, "speed: %d\n", le32_to_cpu(vc.speed)); + seq_printf(seq, "duplex: %d\n", vc.duplex); + seq_printf(seq, "rss_max_key_size: %d\n", vc.rss_max_key_size); + seq_printf(seq, "rss_max_indirection_table_length: %d\n", + le16_to_cpu(vc.rss_max_indirection_table_length)); + seq_printf(seq, "supported_hash_types: %#x\n", + le32_to_cpu(vc.supported_hash_types)); + seq_printf(seq, "vn_status: %#x\n", + __virtio16_to_cpu(true, vc.status)); + + status = vp_modern_get_status(&pdsv->vdpa_aux->vd_mdev); + seq_printf(seq, "dev_status: %#x\n", status); + print_status_bits(seq, status); + + seq_printf(seq, "req_features: %#llx\n", pdsv->req_features); + print_feature_bits_all(seq, pdsv->req_features); + driver_features = vp_modern_get_driver_features(&pdsv->vdpa_aux->vd_mdev); + seq_printf(seq, "driver_features: %#llx\n", driver_features); + print_feature_bits_all(seq, driver_features); + seq_printf(seq, "vdpa_index: %d\n", pdsv->vdpa_index); + seq_printf(seq, "num_vqs: %d\n", pdsv->num_vqs); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(config); + +static int vq_show(struct seq_file *seq, void *v) +{ + struct pds_vdpa_vq_info *vq = seq->private; + + seq_printf(seq, "ready: %d\n", vq->ready); + seq_printf(seq, "desc_addr: %#llx\n", vq->desc_addr); + seq_printf(seq, "avail_addr: %#llx\n", vq->avail_addr); + seq_printf(seq, "used_addr: %#llx\n", vq->used_addr); + seq_printf(seq, "q_len: %d\n", vq->q_len); + seq_printf(seq, "qid: %d\n", vq->qid); + + seq_printf(seq, "doorbell: %#llx\n", vq->doorbell); + seq_printf(seq, "avail_idx: %d\n", vq->avail_idx); + seq_printf(seq, "used_idx: %d\n", vq->used_idx); + seq_printf(seq, "irq: %d\n", vq->irq); + seq_printf(seq, "irq-name: %s\n", vq->irq_name); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(vq); + +void pds_vdpa_debugfs_add_vdpadev(struct pds_vdpa_aux *vdpa_aux) +{ + int i; + + debugfs_create_file("config", 0400, vdpa_aux->dentry, vdpa_aux->pdsv, &config_fops); + + for (i = 0; i < vdpa_aux->pdsv->num_vqs; i++) { + char name[8]; + + snprintf(name, sizeof(name), "vq%02d", i); + debugfs_create_file(name, 0400, vdpa_aux->dentry, + &vdpa_aux->pdsv->vqs[i], &vq_fops); + } +} + +void pds_vdpa_debugfs_del_vdpadev(struct pds_vdpa_aux *vdpa_aux) +{ + debugfs_remove_recursive(vdpa_aux->dentry); + vdpa_aux->dentry = NULL; +} + +void pds_vdpa_debugfs_reset_vdpadev(struct pds_vdpa_aux *vdpa_aux) +{ + /* we don't keep track of the entries, so remove it all + * then rebuild the basics + */ + pds_vdpa_debugfs_del_vdpadev(vdpa_aux); + pds_vdpa_debugfs_add_pcidev(vdpa_aux); + pds_vdpa_debugfs_add_ident(vdpa_aux); +} diff --git a/drivers/vdpa/pds/debugfs.h b/drivers/vdpa/pds/debugfs.h new file mode 100644 index 000000000000..c088a4e8f1e9 --- /dev/null +++ b/drivers/vdpa/pds/debugfs.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ + +#ifndef _PDS_VDPA_DEBUGFS_H_ +#define _PDS_VDPA_DEBUGFS_H_ + +#include <linux/debugfs.h> + +void pds_vdpa_debugfs_create(void); +void pds_vdpa_debugfs_destroy(void); +void pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_aux *vdpa_aux); +void pds_vdpa_debugfs_add_ident(struct pds_vdpa_aux *vdpa_aux); +void pds_vdpa_debugfs_add_vdpadev(struct pds_vdpa_aux *vdpa_aux); +void pds_vdpa_debugfs_del_vdpadev(struct pds_vdpa_aux *vdpa_aux); +void pds_vdpa_debugfs_reset_vdpadev(struct pds_vdpa_aux *vdpa_aux); + +#endif /* _PDS_VDPA_DEBUGFS_H_ */ diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c new file mode 100644 index 000000000000..5071a4d58f8d --- /dev/null +++ b/drivers/vdpa/pds/vdpa_dev.c @@ -0,0 +1,769 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ + +#include <linux/pci.h> +#include <linux/vdpa.h> +#include <uapi/linux/vdpa.h> +#include <linux/virtio_pci_modern.h> + +#include <linux/pds/pds_common.h> +#include <linux/pds/pds_core_if.h> +#include <linux/pds/pds_adminq.h> +#include <linux/pds/pds_auxbus.h> + +#include "vdpa_dev.h" +#include "aux_drv.h" +#include "cmds.h" +#include "debugfs.h" + +static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev); + +static struct pds_vdpa_device *vdpa_to_pdsv(struct vdpa_device *vdpa_dev) +{ + return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev); +} + +static int pds_vdpa_notify_handler(struct notifier_block *nb, + unsigned long ecode, + void *data) +{ + struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device, nb); + struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev; + + dev_dbg(dev, "%s: event code %lu\n", __func__, ecode); + + if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) { + if (pdsv->config_cb.callback) + pdsv->config_cb.callback(pdsv->config_cb.private); + } + + return 0; +} + +static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv) +{ + struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev; + struct notifier_block *nb = &pdsv->nb; + int err; + + if (!nb->notifier_call) { + nb->notifier_call = pds_vdpa_notify_handler; + err = pdsc_register_notify(nb); + if (err) { + nb->notifier_call = NULL; + dev_err(dev, "failed to register pds event handler: %ps\n", + ERR_PTR(err)); + return -EINVAL; + } + dev_dbg(dev, "pds event handler registered\n"); + } + + return 0; +} + +static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv) +{ + if (pdsv->nb.notifier_call) { + pdsc_unregister_notify(&pdsv->nb); + pdsv->nb.notifier_call = NULL; + } +} + +static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, + u64 desc_addr, u64 driver_addr, u64 device_addr) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + pdsv->vqs[qid].desc_addr = desc_addr; + pdsv->vqs[qid].avail_addr = driver_addr; + pdsv->vqs[qid].used_addr = device_addr; + + return 0; +} + +static void pds_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + pdsv->vqs[qid].q_len = num; +} + +static void pds_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + iowrite16(qid, pdsv->vqs[qid].notify); +} + +static void pds_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid, + struct vdpa_callback *cb) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + pdsv->vqs[qid].event_cb = *cb; +} + +static irqreturn_t pds_vdpa_isr(int irq, void *data) +{ + struct pds_vdpa_vq_info *vq; + + vq = data; + if (vq->event_cb.callback) + vq->event_cb.callback(vq->event_cb.private); + + return IRQ_HANDLED; +} + +static void pds_vdpa_release_irq(struct pds_vdpa_device *pdsv, int qid) +{ + if (pdsv->vqs[qid].irq == VIRTIO_MSI_NO_VECTOR) + return; + + free_irq(pdsv->vqs[qid].irq, &pdsv->vqs[qid]); + pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR; +} + +static void pds_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev; + struct device *dev = &pdsv->vdpa_dev.dev; + u64 driver_features; + u16 invert_idx = 0; + int irq; + int err; + + dev_dbg(dev, "%s: qid %d ready %d => %d\n", + __func__, qid, pdsv->vqs[qid].ready, ready); + if (ready == pdsv->vqs[qid].ready) + return; + + driver_features = pds_vdpa_get_driver_features(vdpa_dev); + if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) + invert_idx = PDS_VDPA_PACKED_INVERT_IDX; + + if (ready) { + irq = pci_irq_vector(pdev, qid); + snprintf(pdsv->vqs[qid].irq_name, sizeof(pdsv->vqs[qid].irq_name), + "vdpa-%s-%d", dev_name(dev), qid); + + err = request_irq(irq, pds_vdpa_isr, 0, + pdsv->vqs[qid].irq_name, &pdsv->vqs[qid]); + if (err) { + dev_err(dev, "%s: no irq for qid %d: %pe\n", + __func__, qid, ERR_PTR(err)); + return; + } + pdsv->vqs[qid].irq = irq; + + /* Pass vq setup info to DSC using adminq to gather up and + * send all info at once so FW can do its full set up in + * one easy operation + */ + err = pds_vdpa_cmd_init_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]); + if (err) { + dev_err(dev, "Failed to init vq %d: %pe\n", + qid, ERR_PTR(err)); + pds_vdpa_release_irq(pdsv, qid); + ready = false; + } + } else { + err = pds_vdpa_cmd_reset_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]); + if (err) + dev_err(dev, "%s: reset_vq failed qid %d: %pe\n", + __func__, qid, ERR_PTR(err)); + pds_vdpa_release_irq(pdsv, qid); + } + + pdsv->vqs[qid].ready = ready; +} + +static bool pds_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + return pdsv->vqs[qid].ready; +} + +static int pds_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid, + const struct vdpa_vq_state *state) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; + struct device *dev = &padev->aux_dev.dev; + u64 driver_features; + u16 avail; + u16 used; + + if (pdsv->vqs[qid].ready) { + dev_err(dev, "Setting device position is denied while vq is enabled\n"); + return -EINVAL; + } + + driver_features = pds_vdpa_get_driver_features(vdpa_dev); + if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { + avail = state->packed.last_avail_idx | + (state->packed.last_avail_counter << 15); + used = state->packed.last_used_idx | + (state->packed.last_used_counter << 15); + + /* The avail and used index are stored with the packed wrap + * counter bit inverted. This way, in case set_vq_state is + * not called, the initial value can be set to zero prior to + * feature negotiation, and it is good for both packed and + * split vq. + */ + avail ^= PDS_VDPA_PACKED_INVERT_IDX; + used ^= PDS_VDPA_PACKED_INVERT_IDX; + } else { + avail = state->split.avail_index; + /* state->split does not provide a used_index: + * the vq will be set to "empty" here, and the vq will read + * the current used index the next time the vq is kicked. + */ + used = avail; + } + + if (used != avail) { + dev_dbg(dev, "Setting used equal to avail, for interoperability\n"); + used = avail; + } + + pdsv->vqs[qid].avail_idx = avail; + pdsv->vqs[qid].used_idx = used; + + return 0; +} + +static int pds_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid, + struct vdpa_vq_state *state) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; + struct device *dev = &padev->aux_dev.dev; + u64 driver_features; + u16 avail; + u16 used; + + if (pdsv->vqs[qid].ready) { + dev_err(dev, "Getting device position is denied while vq is enabled\n"); + return -EINVAL; + } + + avail = pdsv->vqs[qid].avail_idx; + used = pdsv->vqs[qid].used_idx; + + driver_features = pds_vdpa_get_driver_features(vdpa_dev); + if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { + avail ^= PDS_VDPA_PACKED_INVERT_IDX; + used ^= PDS_VDPA_PACKED_INVERT_IDX; + + state->packed.last_avail_idx = avail & 0x7fff; + state->packed.last_avail_counter = avail >> 15; + state->packed.last_used_idx = used & 0x7fff; + state->packed.last_used_counter = used >> 15; + } else { + state->split.avail_index = avail; + /* state->split does not provide a used_index. */ + } + + return 0; +} + +static struct vdpa_notification_area +pds_vdpa_get_vq_notification(struct vdpa_device *vdpa_dev, u16 qid) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + struct virtio_pci_modern_device *vd_mdev; + struct vdpa_notification_area area; + + area.addr = pdsv->vqs[qid].notify_pa; + + vd_mdev = &pdsv->vdpa_aux->vd_mdev; + if (!vd_mdev->notify_offset_multiplier) + area.size = PDS_PAGE_SIZE; + else + area.size = vd_mdev->notify_offset_multiplier; + + return area; +} + +static int pds_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, u16 qid) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + return pdsv->vqs[qid].irq; +} + +static u32 pds_vdpa_get_vq_align(struct vdpa_device *vdpa_dev) +{ + return PDS_PAGE_SIZE; +} + +static u32 pds_vdpa_get_vq_group(struct vdpa_device *vdpa_dev, u16 idx) +{ + return 0; +} + +static u64 pds_vdpa_get_device_features(struct vdpa_device *vdpa_dev) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + return pdsv->supported_features; +} + +static int pds_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + struct device *dev = &pdsv->vdpa_dev.dev; + u64 driver_features; + u64 nego_features; + u64 missing; + + if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) { + dev_err(dev, "VIRTIO_F_ACCESS_PLATFORM is not negotiated\n"); + return -EOPNOTSUPP; + } + + pdsv->req_features = features; + + /* Check for valid feature bits */ + nego_features = features & le64_to_cpu(pdsv->vdpa_aux->ident.hw_features); + missing = pdsv->req_features & ~nego_features; + if (missing) { + dev_err(dev, "Can't support all requested features in %#llx, missing %#llx features\n", + pdsv->req_features, missing); + return -EOPNOTSUPP; + } + + driver_features = pds_vdpa_get_driver_features(vdpa_dev); + dev_dbg(dev, "%s: %#llx => %#llx\n", + __func__, driver_features, nego_features); + + if (driver_features == nego_features) + return 0; + + vp_modern_set_features(&pdsv->vdpa_aux->vd_mdev, nego_features); + + return 0; +} + +static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + return vp_modern_get_driver_features(&pdsv->vdpa_aux->vd_mdev); +} + +static void pds_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, + struct vdpa_callback *cb) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + pdsv->config_cb.callback = cb->callback; + pdsv->config_cb.private = cb->private; +} + +static u16 pds_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + /* qemu has assert() that vq_num_max <= VIRTQUEUE_MAX_SIZE (1024) */ + return min_t(u16, 1024, BIT(le16_to_cpu(pdsv->vdpa_aux->ident.max_qlen))); +} + +static u32 pds_vdpa_get_device_id(struct vdpa_device *vdpa_dev) +{ + return VIRTIO_ID_NET; +} + +static u32 pds_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev) +{ + return PCI_VENDOR_ID_PENSANDO; +} + +static u8 pds_vdpa_get_status(struct vdpa_device *vdpa_dev) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + + return vp_modern_get_status(&pdsv->vdpa_aux->vd_mdev); +} + +static void pds_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + struct device *dev = &pdsv->vdpa_dev.dev; + u8 old_status; + int i; + + old_status = pds_vdpa_get_status(vdpa_dev); + dev_dbg(dev, "%s: old %#x new %#x\n", __func__, old_status, status); + + pds_vdpa_cmd_set_status(pdsv, status); + + /* Note: still working with FW on the need for this reset cmd */ + if (status == 0) { + pds_vdpa_cmd_reset(pdsv); + + for (i = 0; i < pdsv->num_vqs; i++) { + pdsv->vqs[i].avail_idx = 0; + pdsv->vqs[i].used_idx = 0; + } + } + + if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) { + for (i = 0; i < pdsv->num_vqs; i++) { + pdsv->vqs[i].notify = + vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev, + i, &pdsv->vqs[i].notify_pa); + } + } +} + +static int pds_vdpa_reset(struct vdpa_device *vdpa_dev) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + struct device *dev; + int err = 0; + u8 status; + int i; + + dev = &pdsv->vdpa_aux->padev->aux_dev.dev; + status = pds_vdpa_get_status(vdpa_dev); + + if (status == 0) + return 0; + + if (status & VIRTIO_CONFIG_S_DRIVER_OK) { + /* Reset the vqs */ + for (i = 0; i < pdsv->num_vqs && !err; i++) { + err = pds_vdpa_cmd_reset_vq(pdsv, i, 0, &pdsv->vqs[i]); + if (err) + dev_err(dev, "%s: reset_vq failed qid %d: %pe\n", + __func__, i, ERR_PTR(err)); + pds_vdpa_release_irq(pdsv, i); + memset(&pdsv->vqs[i], 0, sizeof(pdsv->vqs[0])); + pdsv->vqs[i].ready = false; + } + } + + pds_vdpa_set_status(vdpa_dev, 0); + + return 0; +} + +static size_t pds_vdpa_get_config_size(struct vdpa_device *vdpa_dev) +{ + return sizeof(struct virtio_net_config); +} + +static void pds_vdpa_get_config(struct vdpa_device *vdpa_dev, + unsigned int offset, + void *buf, unsigned int len) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + void __iomem *device; + + if (offset + len > sizeof(struct virtio_net_config)) { + WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len); + return; + } + + device = pdsv->vdpa_aux->vd_mdev.device; + memcpy_fromio(buf, device + offset, len); +} + +static void pds_vdpa_set_config(struct vdpa_device *vdpa_dev, + unsigned int offset, const void *buf, + unsigned int len) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + void __iomem *device; + + if (offset + len > sizeof(struct virtio_net_config)) { + WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len); + return; + } + + device = pdsv->vdpa_aux->vd_mdev.device; + memcpy_toio(device + offset, buf, len); +} + +static const struct vdpa_config_ops pds_vdpa_ops = { + .set_vq_address = pds_vdpa_set_vq_address, + .set_vq_num = pds_vdpa_set_vq_num, + .kick_vq = pds_vdpa_kick_vq, + .set_vq_cb = pds_vdpa_set_vq_cb, + .set_vq_ready = pds_vdpa_set_vq_ready, + .get_vq_ready = pds_vdpa_get_vq_ready, + .set_vq_state = pds_vdpa_set_vq_state, + .get_vq_state = pds_vdpa_get_vq_state, + .get_vq_notification = pds_vdpa_get_vq_notification, + .get_vq_irq = pds_vdpa_get_vq_irq, + .get_vq_align = pds_vdpa_get_vq_align, + .get_vq_group = pds_vdpa_get_vq_group, + + .get_device_features = pds_vdpa_get_device_features, + .set_driver_features = pds_vdpa_set_driver_features, + .get_driver_features = pds_vdpa_get_driver_features, + .set_config_cb = pds_vdpa_set_config_cb, + .get_vq_num_max = pds_vdpa_get_vq_num_max, + .get_device_id = pds_vdpa_get_device_id, + .get_vendor_id = pds_vdpa_get_vendor_id, + .get_status = pds_vdpa_get_status, + .set_status = pds_vdpa_set_status, + .reset = pds_vdpa_reset, + .get_config_size = pds_vdpa_get_config_size, + .get_config = pds_vdpa_get_config, + .set_config = pds_vdpa_set_config, +}; +static struct virtio_device_id pds_vdpa_id_table[] = { + {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID}, + {0}, +}; + +static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, + const struct vdpa_dev_set_config *add_config) +{ + struct pds_vdpa_aux *vdpa_aux; + struct pds_vdpa_device *pdsv; + struct vdpa_mgmt_dev *mgmt; + u16 fw_max_vqs, vq_pairs; + struct device *dma_dev; + struct pci_dev *pdev; + struct device *dev; + u8 mac[ETH_ALEN]; + int err; + int i; + + vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev); + dev = &vdpa_aux->padev->aux_dev.dev; + mgmt = &vdpa_aux->vdpa_mdev; + + if (vdpa_aux->pdsv) { + dev_warn(dev, "Multiple vDPA devices on a VF is not supported.\n"); + return -EOPNOTSUPP; + } + + pdsv = vdpa_alloc_device(struct pds_vdpa_device, vdpa_dev, + dev, &pds_vdpa_ops, 1, 1, name, false); + if (IS_ERR(pdsv)) { + dev_err(dev, "Failed to allocate vDPA structure: %pe\n", pdsv); + return PTR_ERR(pdsv); + } + + vdpa_aux->pdsv = pdsv; + pdsv->vdpa_aux = vdpa_aux; + + pdev = vdpa_aux->padev->vf_pdev; + dma_dev = &pdev->dev; + pdsv->vdpa_dev.dma_dev = dma_dev; + + pdsv->supported_features = mgmt->supported_features; + + if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { + u64 unsupp_features = + add_config->device_features & ~mgmt->supported_features; + + if (unsupp_features) { + dev_err(dev, "Unsupported features: %#llx\n", unsupp_features); + err = -EOPNOTSUPP; + goto err_unmap; + } + + pdsv->supported_features = add_config->device_features; + } + + err = pds_vdpa_cmd_reset(pdsv); + if (err) { + dev_err(dev, "Failed to reset hw: %pe\n", ERR_PTR(err)); + goto err_unmap; + } + + err = pds_vdpa_init_hw(pdsv); + if (err) { + dev_err(dev, "Failed to init hw: %pe\n", ERR_PTR(err)); + goto err_unmap; + } + + fw_max_vqs = le16_to_cpu(pdsv->vdpa_aux->ident.max_vqs); + vq_pairs = fw_max_vqs / 2; + + /* Make sure we have the queues being requested */ + if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MAX_VQP)) + vq_pairs = add_config->net.max_vq_pairs; + + pdsv->num_vqs = 2 * vq_pairs; + if (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) + pdsv->num_vqs++; + + if (pdsv->num_vqs > fw_max_vqs) { + dev_err(dev, "%s: queue count requested %u greater than max %u\n", + __func__, pdsv->num_vqs, fw_max_vqs); + err = -ENOSPC; + goto err_unmap; + } + + if (pdsv->num_vqs != fw_max_vqs) { + err = pds_vdpa_cmd_set_max_vq_pairs(pdsv, vq_pairs); + if (err) { + dev_err(dev, "Failed to set max_vq_pairs: %pe\n", + ERR_PTR(err)); + goto err_unmap; + } + } + + /* Set a mac, either from the user config if provided + * or set a random mac if default is 00:..:00 + */ + if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) { + ether_addr_copy(mac, add_config->net.mac); + pds_vdpa_cmd_set_mac(pdsv, mac); + } else { + struct virtio_net_config __iomem *vc; + + vc = pdsv->vdpa_aux->vd_mdev.device; + memcpy_fromio(mac, vc->mac, sizeof(mac)); + if (is_zero_ether_addr(mac)) { + eth_random_addr(mac); + dev_info(dev, "setting random mac %pM\n", mac); + pds_vdpa_cmd_set_mac(pdsv, mac); + } + } + + for (i = 0; i < pdsv->num_vqs; i++) { + pdsv->vqs[i].qid = i; + pdsv->vqs[i].pdsv = pdsv; + pdsv->vqs[i].irq = VIRTIO_MSI_NO_VECTOR; + pdsv->vqs[i].notify = vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev, + i, &pdsv->vqs[i].notify_pa); + } + + pdsv->vdpa_dev.mdev = &vdpa_aux->vdpa_mdev; + + err = pds_vdpa_register_event_handler(pdsv); + if (err) { + dev_err(dev, "Failed to register for PDS events: %pe\n", ERR_PTR(err)); + goto err_unmap; + } + + /* We use the _vdpa_register_device() call rather than the + * vdpa_register_device() to avoid a deadlock because our + * dev_add() is called with the vdpa_dev_lock already set + * by vdpa_nl_cmd_dev_add_set_doit() + */ + err = _vdpa_register_device(&pdsv->vdpa_dev, pdsv->num_vqs); + if (err) { + dev_err(dev, "Failed to register to vDPA bus: %pe\n", ERR_PTR(err)); + goto err_unevent; + } + + pds_vdpa_debugfs_add_vdpadev(vdpa_aux); + + return 0; + +err_unevent: + pds_vdpa_unregister_event_handler(pdsv); +err_unmap: + put_device(&pdsv->vdpa_dev.dev); + vdpa_aux->pdsv = NULL; + return err; +} + +static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, + struct vdpa_device *vdpa_dev) +{ + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); + struct pds_vdpa_aux *vdpa_aux; + + pds_vdpa_unregister_event_handler(pdsv); + + vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev); + _vdpa_unregister_device(vdpa_dev); + + pds_vdpa_cmd_reset(vdpa_aux->pdsv); + pds_vdpa_debugfs_reset_vdpadev(vdpa_aux); + + vdpa_aux->pdsv = NULL; + + dev_info(&vdpa_aux->padev->aux_dev.dev, "Removed vdpa device\n"); +} + +static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = { + .dev_add = pds_vdpa_dev_add, + .dev_del = pds_vdpa_dev_del +}; + +int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux) +{ + union pds_core_adminq_cmd cmd = { + .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT, + .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id), + }; + union pds_core_adminq_comp comp = {}; + struct vdpa_mgmt_dev *mgmt; + struct pci_dev *pf_pdev; + struct device *pf_dev; + struct pci_dev *pdev; + dma_addr_t ident_pa; + struct device *dev; + u16 dev_intrs; + u16 max_vqs; + int err; + + dev = &vdpa_aux->padev->aux_dev.dev; + pdev = vdpa_aux->padev->vf_pdev; + mgmt = &vdpa_aux->vdpa_mdev; + + /* Get resource info through the PF's adminq. It is a block of info, + * so we need to map some memory for PF to make available to the + * firmware for writing the data. + */ + pf_pdev = pci_physfn(vdpa_aux->padev->vf_pdev); + pf_dev = &pf_pdev->dev; + ident_pa = dma_map_single(pf_dev, &vdpa_aux->ident, + sizeof(vdpa_aux->ident), DMA_FROM_DEVICE); + if (dma_mapping_error(pf_dev, ident_pa)) { + dev_err(dev, "Failed to map ident space\n"); + return -ENOMEM; + } + + cmd.vdpa_ident.ident_pa = cpu_to_le64(ident_pa); + cmd.vdpa_ident.len = cpu_to_le32(sizeof(vdpa_aux->ident)); + err = pds_client_adminq_cmd(vdpa_aux->padev, &cmd, + sizeof(cmd.vdpa_ident), &comp, 0); + dma_unmap_single(pf_dev, ident_pa, + sizeof(vdpa_aux->ident), DMA_FROM_DEVICE); + if (err) { + dev_err(dev, "Failed to ident hw, status %d: %pe\n", + comp.status, ERR_PTR(err)); + return err; + } + + max_vqs = le16_to_cpu(vdpa_aux->ident.max_vqs); + dev_intrs = pci_msix_vec_count(pdev); + dev_dbg(dev, "ident.max_vqs %d dev_intrs %d\n", max_vqs, dev_intrs); + + max_vqs = min_t(u16, dev_intrs, max_vqs); + mgmt->max_supported_vqs = min_t(u16, PDS_VDPA_MAX_QUEUES, max_vqs); + vdpa_aux->nintrs = mgmt->max_supported_vqs; + + mgmt->ops = &pds_vdpa_mgmt_dev_ops; + mgmt->id_table = pds_vdpa_id_table; + mgmt->device = dev; + mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features); + mgmt->config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR); + mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP); + mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES); + + err = pci_alloc_irq_vectors(pdev, vdpa_aux->nintrs, vdpa_aux->nintrs, + PCI_IRQ_MSIX); + if (err < 0) { + dev_err(dev, "Couldn't get %d msix vectors: %pe\n", + vdpa_aux->nintrs, ERR_PTR(err)); + return err; + } + vdpa_aux->nintrs = err; + + return 0; +} diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h new file mode 100644 index 000000000000..a1bc37de9537 --- /dev/null +++ b/drivers/vdpa/pds/vdpa_dev.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ + +#ifndef _VDPA_DEV_H_ +#define _VDPA_DEV_H_ + +#include <linux/pci.h> +#include <linux/vdpa.h> + +struct pds_vdpa_vq_info { + bool ready; + u64 desc_addr; + u64 avail_addr; + u64 used_addr; + u32 q_len; + u16 qid; + int irq; + char irq_name[32]; + + void __iomem *notify; + dma_addr_t notify_pa; + + u64 doorbell; + u16 avail_idx; + u16 used_idx; + + struct vdpa_callback event_cb; + struct pds_vdpa_device *pdsv; +}; + +#define PDS_VDPA_MAX_QUEUES 65 +#define PDS_VDPA_MAX_QLEN 32768 +struct pds_vdpa_device { + struct vdpa_device vdpa_dev; + struct pds_vdpa_aux *vdpa_aux; + + struct pds_vdpa_vq_info vqs[PDS_VDPA_MAX_QUEUES]; + u64 supported_features; /* specified device features */ + u64 req_features; /* features requested by vdpa */ + u8 vdpa_index; /* rsvd for future subdevice use */ + u8 num_vqs; /* num vqs in use */ + struct vdpa_callback config_cb; + struct notifier_block nb; +}; + +#define PDS_VDPA_PACKED_INVERT_IDX 0x8000 + +int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux); +#endif /* _VDPA_DEV_H_ */ diff --git a/drivers/vdpa/solidrun/snet_ctrl.c b/drivers/vdpa/solidrun/snet_ctrl.c index 3858738643b4..3cef2571d15d 100644 --- a/drivers/vdpa/solidrun/snet_ctrl.c +++ b/drivers/vdpa/solidrun/snet_ctrl.c @@ -16,6 +16,7 @@ enum snet_ctrl_opcodes { SNET_CTRL_OP_DESTROY = 1, SNET_CTRL_OP_READ_VQ_STATE, SNET_CTRL_OP_SUSPEND, + SNET_CTRL_OP_RESUME, }; #define SNET_CTRL_TIMEOUT 2000000 @@ -328,3 +329,8 @@ int snet_suspend_dev(struct snet *snet) { return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0); } + +int snet_resume_dev(struct snet *snet) +{ + return snet_send_ctrl_msg(snet, SNET_CTRL_OP_RESUME, 0); +} diff --git a/drivers/vdpa/solidrun/snet_hwmon.c b/drivers/vdpa/solidrun/snet_hwmon.c index 42c87387a0f1..af531a339082 100644 --- a/drivers/vdpa/solidrun/snet_hwmon.c +++ b/drivers/vdpa/solidrun/snet_hwmon.c @@ -159,7 +159,7 @@ static const struct hwmon_ops snet_hwmon_ops = { .read_string = snet_hwmon_read_string }; -static const struct hwmon_channel_info *snet_hwmon_info[] = { +static const struct hwmon_channel_info * const snet_hwmon_info[] = { HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_CRIT | HWMON_T_LABEL), HWMON_CHANNEL_INFO(power, HWMON_P_INPUT | HWMON_P_LABEL), diff --git a/drivers/vdpa/solidrun/snet_main.c b/drivers/vdpa/solidrun/snet_main.c index cdcd84ce4f5a..99428a04068d 100644 --- a/drivers/vdpa/solidrun/snet_main.c +++ b/drivers/vdpa/solidrun/snet_main.c @@ -509,6 +509,20 @@ static int snet_suspend(struct vdpa_device *vdev) return ret; } +static int snet_resume(struct vdpa_device *vdev) +{ + struct snet *snet = vdpa_to_snet(vdev); + int ret; + + ret = snet_resume_dev(snet); + if (ret) + SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n", snet->sid, ret); + else + SNET_DBG(snet->pdev, "Resume SNET[%u] device\n", snet->sid); + + return ret; +} + static const struct vdpa_config_ops snet_config_ops = { .set_vq_address = snet_set_vq_address, .set_vq_num = snet_set_vq_num, @@ -536,6 +550,7 @@ static const struct vdpa_config_ops snet_config_ops = { .get_config = snet_get_config, .set_config = snet_set_config, .suspend = snet_suspend, + .resume = snet_resume, }; static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet) diff --git a/drivers/vdpa/solidrun/snet_vdpa.h b/drivers/vdpa/solidrun/snet_vdpa.h index 3c78d4e7d485..36ac285835ea 100644 --- a/drivers/vdpa/solidrun/snet_vdpa.h +++ b/drivers/vdpa/solidrun/snet_vdpa.h @@ -204,5 +204,6 @@ void snet_ctrl_clear(struct snet *snet); int snet_destroy_dev(struct snet *snet); int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state); int snet_suspend_dev(struct snet *snet); +int snet_resume_dev(struct snet *snet); #endif //_SNET_VDPA_H_ diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index 4619b4a520ef..dc38ed21319d 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -726,7 +726,11 @@ static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx, { struct vduse_dev *dev = vdpa_to_vduse(vdpa); - cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask); + if (cpu_mask) + cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask); + else + cpumask_setall(&dev->vqs[idx]->irq_affinity); + return 0; } |