summaryrefslogtreecommitdiff
path: root/drivers/pci/msi.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/pci/msi.c')
-rw-r--r--drivers/pci/msi.c304
1 files changed, 99 insertions, 205 deletions
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 3a9f4f8ad8f9..0099a00af361 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -129,94 +129,95 @@ void __weak arch_restore_msi_irqs(struct pci_dev *dev)
return default_restore_msi_irqs(dev);
}
-static inline __attribute_const__ u32 msi_mask(unsigned x)
-{
- /* Don't shift by >= width of type */
- if (x >= 5)
- return 0xffffffff;
- return (1 << (1 << x)) - 1;
-}
-
/*
* PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to
* mask all MSI interrupts by clearing the MSI enable bit does not work
* reliably as devices without an INTx disable bit will then generate a
* level IRQ which will never be cleared.
*/
-void __pci_msi_desc_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
+static inline __attribute_const__ u32 msi_multi_mask(struct msi_desc *desc)
+{
+ /* Don't shift by >= width of type */
+ if (desc->msi_attrib.multi_cap >= 5)
+ return 0xffffffff;
+ return (1 << (1 << desc->msi_attrib.multi_cap)) - 1;
+}
+
+static noinline void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 set)
{
raw_spinlock_t *lock = &desc->dev->msi_lock;
unsigned long flags;
- if (pci_msi_ignore_mask || !desc->msi_attrib.maskbit)
- return;
-
raw_spin_lock_irqsave(lock, flags);
- desc->masked &= ~mask;
- desc->masked |= flag;
+ desc->msi_mask &= ~clear;
+ desc->msi_mask |= set;
pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos,
- desc->masked);
+ desc->msi_mask);
raw_spin_unlock_irqrestore(lock, flags);
}
-static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
+static inline void pci_msi_mask(struct msi_desc *desc, u32 mask)
{
- __pci_msi_desc_mask_irq(desc, mask, flag);
+ pci_msi_update_mask(desc, 0, mask);
}
-static void __iomem *pci_msix_desc_addr(struct msi_desc *desc)
+static inline void pci_msi_unmask(struct msi_desc *desc, u32 mask)
{
- if (desc->msi_attrib.is_virtual)
- return NULL;
+ pci_msi_update_mask(desc, mask, 0);
+}
- return desc->mask_base +
- desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
+static inline void __iomem *pci_msix_desc_addr(struct msi_desc *desc)
+{
+ return desc->mask_base + desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
}
/*
- * This internal function does not flush PCI writes to the device.
- * All users must ensure that they read from the device before either
- * assuming that the device state is up to date, or returning out of this
- * file. This saves a few milliseconds when initialising devices with lots
- * of MSI-X interrupts.
+ * This internal function does not flush PCI writes to the device. All
+ * users must ensure that they read from the device before either assuming
+ * that the device state is up to date, or returning out of this file.
+ * It does not affect the msi_desc::msix_ctrl cache either. Use with care!
*/
-u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag)
+static void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl)
{
- u32 mask_bits = desc->masked;
- void __iomem *desc_addr;
-
- if (pci_msi_ignore_mask)
- return 0;
-
- desc_addr = pci_msix_desc_addr(desc);
- if (!desc_addr)
- return 0;
+ void __iomem *desc_addr = pci_msix_desc_addr(desc);
- mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
- if (flag & PCI_MSIX_ENTRY_CTRL_MASKBIT)
- mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
+ writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
+}
- writel(mask_bits, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
+static inline void pci_msix_mask(struct msi_desc *desc)
+{
+ desc->msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
+ pci_msix_write_vector_ctrl(desc, desc->msix_ctrl);
+ /* Flush write to device */
+ readl(desc->mask_base);
+}
- return mask_bits;
+static inline void pci_msix_unmask(struct msi_desc *desc)
+{
+ desc->msix_ctrl &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
+ pci_msix_write_vector_ctrl(desc, desc->msix_ctrl);
}
-static void msix_mask_irq(struct msi_desc *desc, u32 flag)
+static void __pci_msi_mask_desc(struct msi_desc *desc, u32 mask)
{
- desc->masked = __pci_msix_desc_mask_irq(desc, flag);
+ if (pci_msi_ignore_mask || desc->msi_attrib.is_virtual)
+ return;
+
+ if (desc->msi_attrib.is_msix)
+ pci_msix_mask(desc);
+ else if (desc->msi_attrib.maskbit)
+ pci_msi_mask(desc, mask);
}
-static void msi_set_mask_bit(struct irq_data *data, u32 flag)
+static void __pci_msi_unmask_desc(struct msi_desc *desc, u32 mask)
{
- struct msi_desc *desc = irq_data_get_msi_desc(data);
+ if (pci_msi_ignore_mask || desc->msi_attrib.is_virtual)
+ return;
- if (desc->msi_attrib.is_msix) {
- msix_mask_irq(desc, flag);
- readl(desc->mask_base); /* Flush write to device */
- } else {
- unsigned offset = data->irq - desc->irq;
- msi_mask_irq(desc, 1 << offset, flag << offset);
- }
+ if (desc->msi_attrib.is_msix)
+ pci_msix_unmask(desc);
+ else if (desc->msi_attrib.maskbit)
+ pci_msi_unmask(desc, mask);
}
/**
@@ -225,7 +226,9 @@ static void msi_set_mask_bit(struct irq_data *data, u32 flag)
*/
void pci_msi_mask_irq(struct irq_data *data)
{
- msi_set_mask_bit(data, 1);
+ struct msi_desc *desc = irq_data_get_msi_desc(data);
+
+ __pci_msi_mask_desc(desc, BIT(data->irq - desc->irq));
}
EXPORT_SYMBOL_GPL(pci_msi_mask_irq);
@@ -235,7 +238,9 @@ EXPORT_SYMBOL_GPL(pci_msi_mask_irq);
*/
void pci_msi_unmask_irq(struct irq_data *data)
{
- msi_set_mask_bit(data, 0);
+ struct msi_desc *desc = irq_data_get_msi_desc(data);
+
+ __pci_msi_unmask_desc(desc, BIT(data->irq - desc->irq));
}
EXPORT_SYMBOL_GPL(pci_msi_unmask_irq);
@@ -256,10 +261,8 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
if (entry->msi_attrib.is_msix) {
void __iomem *base = pci_msix_desc_addr(entry);
- if (!base) {
- WARN_ON(1);
+ if (WARN_ON_ONCE(entry->msi_attrib.is_virtual))
return;
- }
msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR);
msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR);
@@ -290,9 +293,10 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
/* Don't touch the hardware now */
} else if (entry->msi_attrib.is_msix) {
void __iomem *base = pci_msix_desc_addr(entry);
- bool unmasked = !(entry->masked & PCI_MSIX_ENTRY_CTRL_MASKBIT);
+ u32 ctrl = entry->msix_ctrl;
+ bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT);
- if (!base)
+ if (entry->msi_attrib.is_virtual)
goto skip;
/*
@@ -304,14 +308,14 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
* undefined."
*/
if (unmasked)
- __pci_msix_desc_mask_irq(entry, PCI_MSIX_ENTRY_CTRL_MASKBIT);
+ pci_msix_write_vector_ctrl(entry, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT);
writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR);
writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR);
writel(msg->data, base + PCI_MSIX_ENTRY_DATA);
if (unmasked)
- __pci_msix_desc_mask_irq(entry, 0);
+ pci_msix_write_vector_ctrl(entry, ctrl);
/* Ensure that the writes are visible in the device */
readl(base + PCI_MSIX_ENTRY_DATA);
@@ -359,9 +363,7 @@ static void free_msi_irqs(struct pci_dev *dev)
{
struct list_head *msi_list = dev_to_msi_list(&dev->dev);
struct msi_desc *entry, *tmp;
- struct attribute **msi_attrs;
- struct device_attribute *dev_attr;
- int i, count = 0;
+ int i;
for_each_pci_msi_entry(entry, dev)
if (entry->irq)
@@ -381,18 +383,7 @@ static void free_msi_irqs(struct pci_dev *dev)
}
if (dev->msi_irq_groups) {
- sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups);
- msi_attrs = dev->msi_irq_groups[0]->attrs;
- while (msi_attrs[count]) {
- dev_attr = container_of(msi_attrs[count],
- struct device_attribute, attr);
- kfree(dev_attr->attr.name);
- kfree(dev_attr);
- ++count;
- }
- kfree(msi_attrs);
- kfree(dev->msi_irq_groups[0]);
- kfree(dev->msi_irq_groups);
+ msi_destroy_sysfs(&dev->dev, dev->msi_irq_groups);
dev->msi_irq_groups = NULL;
}
}
@@ -429,8 +420,7 @@ static void __pci_restore_msi_state(struct pci_dev *dev)
arch_restore_msi_irqs(dev);
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
- msi_mask_irq(entry, msi_mask(entry->msi_attrib.multi_cap),
- entry->masked);
+ pci_msi_update_mask(entry, 0, 0);
control &= ~PCI_MSI_FLAGS_QSIZE;
control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE;
pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
@@ -461,7 +451,7 @@ static void __pci_restore_msix_state(struct pci_dev *dev)
arch_restore_msi_irqs(dev);
for_each_pci_msi_entry(entry, dev)
- msix_mask_irq(entry, entry->masked);
+ pci_msix_write_vector_ctrl(entry, entry->msix_ctrl);
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
}
@@ -473,102 +463,6 @@ void pci_restore_msi_state(struct pci_dev *dev)
}
EXPORT_SYMBOL_GPL(pci_restore_msi_state);
-static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
- char *buf)
-{
- struct msi_desc *entry;
- unsigned long irq;
- int retval;
-
- retval = kstrtoul(attr->attr.name, 10, &irq);
- if (retval)
- return retval;
-
- entry = irq_get_msi_desc(irq);
- if (!entry)
- return -ENODEV;
-
- return sysfs_emit(buf, "%s\n",
- entry->msi_attrib.is_msix ? "msix" : "msi");
-}
-
-static int populate_msi_sysfs(struct pci_dev *pdev)
-{
- struct attribute **msi_attrs;
- struct attribute *msi_attr;
- struct device_attribute *msi_dev_attr;
- struct attribute_group *msi_irq_group;
- const struct attribute_group **msi_irq_groups;
- struct msi_desc *entry;
- int ret = -ENOMEM;
- int num_msi = 0;
- int count = 0;
- int i;
-
- /* Determine how many msi entries we have */
- for_each_pci_msi_entry(entry, pdev)
- num_msi += entry->nvec_used;
- if (!num_msi)
- return 0;
-
- /* Dynamically create the MSI attributes for the PCI device */
- msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL);
- if (!msi_attrs)
- return -ENOMEM;
- for_each_pci_msi_entry(entry, pdev) {
- for (i = 0; i < entry->nvec_used; i++) {
- msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
- if (!msi_dev_attr)
- goto error_attrs;
- msi_attrs[count] = &msi_dev_attr->attr;
-
- sysfs_attr_init(&msi_dev_attr->attr);
- msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d",
- entry->irq + i);
- if (!msi_dev_attr->attr.name)
- goto error_attrs;
- msi_dev_attr->attr.mode = S_IRUGO;
- msi_dev_attr->show = msi_mode_show;
- ++count;
- }
- }
-
- msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
- if (!msi_irq_group)
- goto error_attrs;
- msi_irq_group->name = "msi_irqs";
- msi_irq_group->attrs = msi_attrs;
-
- msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL);
- if (!msi_irq_groups)
- goto error_irq_group;
- msi_irq_groups[0] = msi_irq_group;
-
- ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups);
- if (ret)
- goto error_irq_groups;
- pdev->msi_irq_groups = msi_irq_groups;
-
- return 0;
-
-error_irq_groups:
- kfree(msi_irq_groups);
-error_irq_group:
- kfree(msi_irq_group);
-error_attrs:
- count = 0;
- msi_attr = msi_attrs[count];
- while (msi_attr) {
- msi_dev_attr = container_of(msi_attr, struct device_attribute, attr);
- kfree(msi_attr->name);
- kfree(msi_dev_attr);
- ++count;
- msi_attr = msi_attrs[count];
- }
- kfree(msi_attrs);
- return ret;
-}
-
static struct msi_desc *
msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
{
@@ -602,7 +496,7 @@ msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
/* Save the initial mask status */
if (entry->msi_attrib.maskbit)
- pci_read_config_dword(dev, entry->mask_pos, &entry->masked);
+ pci_read_config_dword(dev, entry->mask_pos, &entry->msi_mask);
out:
kfree(masks);
@@ -613,8 +507,11 @@ static int msi_verify_entries(struct pci_dev *dev)
{
struct msi_desc *entry;
+ if (!dev->no_64bit_msi)
+ return 0;
+
for_each_pci_msi_entry(entry, dev) {
- if (entry->msg.address_hi && dev->no_64bit_msi) {
+ if (entry->msg.address_hi) {
pci_err(dev, "arch assigned 64-bit MSI address %#x%08x but device only supports 32 bits\n",
entry->msg.address_hi, entry->msg.address_lo);
return -EIO;
@@ -640,7 +537,6 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
{
struct msi_desc *entry;
int ret;
- unsigned mask;
pci_msi_set_enable(dev, 0); /* Disable MSI during set up */
@@ -649,31 +545,23 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
return -ENOMEM;
/* All MSIs are unmasked by default; mask them all */
- mask = msi_mask(entry->msi_attrib.multi_cap);
- msi_mask_irq(entry, mask, mask);
+ pci_msi_mask(entry, msi_multi_mask(entry));
list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
/* Configure MSI capability structure */
ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI);
- if (ret) {
- msi_mask_irq(entry, mask, 0);
- free_msi_irqs(dev);
- return ret;
- }
+ if (ret)
+ goto err;
ret = msi_verify_entries(dev);
- if (ret) {
- msi_mask_irq(entry, mask, 0);
- free_msi_irqs(dev);
- return ret;
- }
+ if (ret)
+ goto err;
- ret = populate_msi_sysfs(dev);
- if (ret) {
- msi_mask_irq(entry, mask, 0);
- free_msi_irqs(dev);
- return ret;
+ dev->msi_irq_groups = msi_populate_sysfs(&dev->dev);
+ if (IS_ERR(dev->msi_irq_groups)) {
+ ret = PTR_ERR(dev->msi_irq_groups);
+ goto err;
}
/* Set MSI enabled bits */
@@ -684,6 +572,11 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
pcibios_free_irq(dev);
dev->irq = entry->irq;
return 0;
+
+err:
+ pci_msi_unmask(entry, msi_multi_mask(entry));
+ free_msi_irqs(dev);
+ return ret;
}
static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries)
@@ -745,9 +638,10 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
entry->msi_attrib.default_irq = dev->irq;
entry->mask_base = base;
- addr = pci_msix_desc_addr(entry);
- if (addr)
- entry->masked = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
+ if (!entry->msi_attrib.is_virtual) {
+ addr = pci_msix_desc_addr(entry);
+ entry->msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
+ }
list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
if (masks)
@@ -836,9 +730,11 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
msix_update_entries(dev, entries);
- ret = populate_msi_sysfs(dev);
- if (ret)
+ dev->msi_irq_groups = msi_populate_sysfs(&dev->dev);
+ if (IS_ERR(dev->msi_irq_groups)) {
+ ret = PTR_ERR(dev->msi_irq_groups);
goto out_free;
+ }
/* Set MSI-X enabled bits and unmask the function */
pci_intx_for_msi(dev, 0);
@@ -951,7 +847,6 @@ EXPORT_SYMBOL(pci_msi_vec_count);
static void pci_msi_shutdown(struct pci_dev *dev)
{
struct msi_desc *desc;
- u32 mask;
if (!pci_msi_enable || !dev || !dev->msi_enabled)
return;
@@ -964,8 +859,7 @@ static void pci_msi_shutdown(struct pci_dev *dev)
dev->msi_enabled = 0;
/* Return the device with MSI unmasked as initial states */
- mask = msi_mask(desc->msi_attrib.multi_cap);
- msi_mask_irq(desc, mask, 0);
+ pci_msi_unmask(desc, msi_multi_mask(desc));
/* Restore dev->irq to its default pin-assertion IRQ */
dev->irq = desc->msi_attrib.default_irq;
@@ -1051,7 +945,7 @@ static void pci_msix_shutdown(struct pci_dev *dev)
/* Return the device with MSI-X masked as initial states */
for_each_pci_msi_entry(entry, dev)
- __pci_msix_desc_mask_irq(entry, 1);
+ pci_msix_mask(entry);
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
pci_intx_for_msi(dev, 1);