diff options
author | Tariq Toukan <tariqt@nvidia.com> | 2023-12-10 17:24:36 +0300 |
---|---|---|
committer | Saeed Mahameed <saeedm@nvidia.com> | 2023-12-21 03:54:25 +0300 |
commit | a45af9a96740873db9a4b5bb493ce2ad81ccb4d5 (patch) | |
tree | 541a2c91fff8690e5b2a4cbc043f53ccb2cef838 /drivers/net/ethernet/mellanox/mlx5/core/lib | |
parent | 63b9ce944c0e26c44c42cdd5095c2e9851c1a8ff (diff) | |
download | linux-a45af9a96740873db9a4b5bb493ce2ad81ccb4d5.tar.xz |
net/mlx5: SD, Implement devcom communication and primary election
Use devcom to communicate between the different devices. Add a new
devcom component type for this.
Each device registers itself to the devcom component <SD, group ID>.
Once all devices of a component are registered, the component becomes
ready, and a primary device is elected.
In principle, any of the devices can act as a primary, they are all
capable, and a random election would've worked. However, we aim to
achieve predictability and consistency, hence each group always choses
the same device, with the lowest PCI BUS number, as primary.
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
Diffstat (limited to 'drivers/net/ethernet/mellanox/mlx5/core/lib')
-rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h | 1 | ||||
-rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c | 122 |
2 files changed, 121 insertions, 2 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h index ec32b686f586..d58032dd0df7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h @@ -10,6 +10,7 @@ enum mlx5_devcom_component { MLX5_DEVCOM_ESW_OFFLOADS, MLX5_DEVCOM_MPV, MLX5_DEVCOM_HCA_PORTS, + MLX5_DEVCOM_SD_GROUP, MLX5_DEVCOM_NUM_COMPONENTS, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c index 9d8b1bb0c0a6..19e674dd1af7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c @@ -14,6 +14,16 @@ struct mlx5_sd { u32 group_id; u8 host_buses; + struct mlx5_devcom_comp_dev *devcom; + bool primary; + union { + struct { /* primary */ + struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1]; + }; + struct { /* secondary */ + struct mlx5_core_dev *primary_dev; + }; + }; }; static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev) @@ -26,13 +36,29 @@ static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev) return sd->host_buses; } +static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + + if (!sd) + return dev; + + return sd->primary ? dev : sd->primary_dev; +} + struct mlx5_core_dev * mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx) { + struct mlx5_sd *sd; + if (idx == 0) return primary; - return NULL; + if (idx >= mlx5_sd_get_host_buses(primary)) + return NULL; + + sd = mlx5_get_sd(primary); + return sd->secondaries[idx - 1]; } int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix) @@ -136,15 +162,93 @@ static void sd_cleanup(struct mlx5_core_dev *dev) kfree(sd); } +static int sd_register(struct mlx5_core_dev *dev) +{ + struct mlx5_devcom_comp_dev *devcom, *pos; + struct mlx5_core_dev *peer, *primary; + struct mlx5_sd *sd, *primary_sd; + int err, i; + + sd = mlx5_get_sd(dev); + devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP, + sd->group_id, NULL, dev); + if (!devcom) + return -ENOMEM; + + sd->devcom = devcom; + + if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses) + return 0; + + mlx5_devcom_comp_lock(devcom); + mlx5_devcom_comp_set_ready(devcom, true); + mlx5_devcom_comp_unlock(devcom); + + if (!mlx5_devcom_for_each_peer_begin(devcom)) { + err = -ENODEV; + goto err_devcom_unreg; + } + + primary = dev; + mlx5_devcom_for_each_peer_entry(devcom, peer, pos) + if (peer->pdev->bus->number < primary->pdev->bus->number) + primary = peer; + + primary_sd = mlx5_get_sd(primary); + primary_sd->primary = true; + i = 0; + /* loop the secondaries */ + mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) { + struct mlx5_sd *peer_sd = mlx5_get_sd(peer); + + primary_sd->secondaries[i++] = peer; + peer_sd->primary = false; + peer_sd->primary_dev = primary; + } + + mlx5_devcom_for_each_peer_end(devcom); + return 0; + +err_devcom_unreg: + mlx5_devcom_comp_lock(sd->devcom); + mlx5_devcom_comp_set_ready(sd->devcom, false); + mlx5_devcom_comp_unlock(sd->devcom); + mlx5_devcom_unregister_component(sd->devcom); + return err; +} + +static void sd_unregister(struct mlx5_core_dev *dev) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + + mlx5_devcom_comp_lock(sd->devcom); + mlx5_devcom_comp_set_ready(sd->devcom, false); + mlx5_devcom_comp_unlock(sd->devcom); + mlx5_devcom_unregister_component(sd->devcom); +} + int mlx5_sd_init(struct mlx5_core_dev *dev) { + struct mlx5_sd *sd = mlx5_get_sd(dev); int err; err = sd_init(dev); if (err) return err; + sd = mlx5_get_sd(dev); + if (!sd) + return 0; + + err = sd_register(dev); + if (err) + goto err_sd_cleanup; + return 0; + +err_sd_cleanup: + sd_cleanup(dev); + return err; } void mlx5_sd_cleanup(struct mlx5_core_dev *dev) @@ -154,6 +258,7 @@ void mlx5_sd_cleanup(struct mlx5_core_dev *dev) if (!sd) return; + sd_unregister(dev); sd_cleanup(dev); } @@ -161,5 +266,18 @@ struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev, struct auxiliary_device *adev, int idx) { - return adev; + struct mlx5_sd *sd = mlx5_get_sd(dev); + struct mlx5_core_dev *primary; + + if (!sd) + return adev; + + if (!mlx5_devcom_comp_is_ready(sd->devcom)) + return NULL; + + primary = mlx5_sd_get_primary(dev); + if (dev == primary) + return adev; + + return &primary->priv.adev[idx]->adev; } |