diff options
Diffstat (limited to 'drivers')
700 files changed, 26576 insertions, 7017 deletions
diff --git a/drivers/Makefile b/drivers/Makefile index 3bf5cab4b451..fe9ceb0d2288 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -6,11 +6,6 @@ # Rewritten to use lists instead of if-statements. # -# Some driver Makefiles miss $(srctree)/ for include directive. -ifdef building_out_of_srctree -MAKEFLAGS += --include-dir=$(srctree) -endif - obj-y += cache/ obj-y += irqchip/ obj-y += bus/ diff --git a/drivers/accel/ivpu/ivpu_mmu_context.c b/drivers/accel/ivpu/ivpu_mmu_context.c index fe6161299236..128aef8e5a19 100644 --- a/drivers/accel/ivpu/ivpu_mmu_context.c +++ b/drivers/accel/ivpu/ivpu_mmu_context.c @@ -6,6 +6,7 @@ #include <linux/bitfield.h> #include <linux/highmem.h> #include <linux/set_memory.h> +#include <linux/vmalloc.h> #include <drm/drm_cache.h> diff --git a/drivers/acpi/arm64/dma.c b/drivers/acpi/arm64/dma.c index 93d796531af3..52b2abf88689 100644 --- a/drivers/acpi/arm64/dma.c +++ b/drivers/acpi/arm64/dma.c @@ -8,7 +8,6 @@ void acpi_arch_dma_setup(struct device *dev) { int ret; u64 end, mask; - u64 size = 0; const struct bus_dma_region *map = NULL; /* @@ -23,31 +22,23 @@ void acpi_arch_dma_setup(struct device *dev) } if (dev->coherent_dma_mask) - size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1); + end = dev->coherent_dma_mask; else - size = 1ULL << 32; + end = (1ULL << 32) - 1; ret = acpi_dma_get_range(dev, &map); if (!ret && map) { - const struct bus_dma_region *r = map; - - for (end = 0; r->size; r++) { - if (r->dma_start + r->size - 1 > end) - end = r->dma_start + r->size - 1; - } - - size = end + 1; + end = dma_range_map_max(map); dev->dma_range_map = map; } if (ret == -ENODEV) - ret = iort_dma_get_ranges(dev, &size); + ret = iort_dma_get_ranges(dev, &end); if (!ret) { /* * Limit coherent and dma mask based on size retrieved from * firmware. */ - end = size - 1; mask = DMA_BIT_MASK(ilog2(end) + 1); dev->bus_dma_limit = end; dev->coherent_dma_mask = min(dev->coherent_dma_mask, mask); diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 6496ff5a6ba2..c0b1c2c19444 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -1367,7 +1367,7 @@ int iort_iommu_configure_id(struct device *dev, const u32 *input_id) { return -ENODEV; } #endif -static int nc_dma_get_range(struct device *dev, u64 *size) +static int nc_dma_get_range(struct device *dev, u64 *limit) { struct acpi_iort_node *node; struct acpi_iort_named_component *ncomp; @@ -1384,13 +1384,13 @@ static int nc_dma_get_range(struct device *dev, u64 *size) return -EINVAL; } - *size = ncomp->memory_address_limit >= 64 ? U64_MAX : - 1ULL<<ncomp->memory_address_limit; + *limit = ncomp->memory_address_limit >= 64 ? U64_MAX : + (1ULL << ncomp->memory_address_limit) - 1; return 0; } -static int rc_dma_get_range(struct device *dev, u64 *size) +static int rc_dma_get_range(struct device *dev, u64 *limit) { struct acpi_iort_node *node; struct acpi_iort_root_complex *rc; @@ -1408,8 +1408,8 @@ static int rc_dma_get_range(struct device *dev, u64 *size) return -EINVAL; } - *size = rc->memory_address_limit >= 64 ? U64_MAX : - 1ULL<<rc->memory_address_limit; + *limit = rc->memory_address_limit >= 64 ? U64_MAX : + (1ULL << rc->memory_address_limit) - 1; return 0; } @@ -1417,16 +1417,16 @@ static int rc_dma_get_range(struct device *dev, u64 *size) /** * iort_dma_get_ranges() - Look up DMA addressing limit for the device * @dev: device to lookup - * @size: DMA range size result pointer + * @limit: DMA limit result pointer * * Return: 0 on success, an error otherwise. */ -int iort_dma_get_ranges(struct device *dev, u64 *size) +int iort_dma_get_ranges(struct device *dev, u64 *limit) { if (dev_is_pci(dev)) - return rc_dma_get_range(dev, size); + return rc_dma_get_range(dev, limit); else - return nc_dma_get_range(dev, size); + return nc_dma_get_range(dev, limit); } static void __init acpi_iort_register_irq(int hwirq, const char *name, diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index e41fb5716bc0..503773707e01 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -1675,12 +1675,7 @@ int acpi_dma_configure_id(struct device *dev, enum dev_dma_attr attr, if (ret == -EPROBE_DEFER) return -EPROBE_DEFER; - /* - * Historically this routine doesn't fail driver probing due to errors - * in acpi_iommu_configure_id(). - */ - - arch_setup_dma_ops(dev, 0, U64_MAX, attr == DEV_DMA_COHERENT); + arch_setup_dma_ops(dev, attr == DEV_DMA_COHERENT); return 0; } diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index a24c152bfaac..aba3aa95b224 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -488,28 +488,31 @@ static int __init amba_stub_drv_init(void) * waiting on amba_match(). So, register a stub driver to make sure * amba_match() is called even if no amba driver has been registered. */ - return amba_driver_register(&amba_proxy_drv); + return __amba_driver_register(&amba_proxy_drv, NULL); } late_initcall_sync(amba_stub_drv_init); /** - * amba_driver_register - register an AMBA device driver + * __amba_driver_register - register an AMBA device driver * @drv: amba device driver structure + * @owner: owning module/driver * * Register an AMBA device driver with the Linux device model * core. If devices pre-exist, the drivers probe function will * be called. */ -int amba_driver_register(struct amba_driver *drv) +int __amba_driver_register(struct amba_driver *drv, + struct module *owner) { if (!drv->probe) return -EINVAL; + drv->drv.owner = owner; drv->drv.bus = &amba_bustype; return driver_register(&drv->drv); } -EXPORT_SYMBOL(amba_driver_register); +EXPORT_SYMBOL(__amba_driver_register); /** * amba_driver_unregister - remove an AMBA device driver diff --git a/drivers/ata/pata_cs5520.c b/drivers/ata/pata_cs5520.c index 38795508c2e9..027cf67101ef 100644 --- a/drivers/ata/pata_cs5520.c +++ b/drivers/ata/pata_cs5520.c @@ -151,12 +151,6 @@ static int cs5520_init_one(struct pci_dev *pdev, const struct pci_device_id *id) if (!host) return -ENOMEM; - /* Perform set up for DMA */ - if (pci_enable_device_io(pdev)) { - dev_err(&pdev->dev, "unable to configure BAR2.\n"); - return -ENODEV; - } - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) { dev_err(&pdev->dev, "unable to configure DMA mask.\n"); return -ENODEV; diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 0248912ff687..c66d070207a0 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -179,7 +179,7 @@ DEFINE_PER_CPU(unsigned long, hw_pressure); void topology_update_hw_pressure(const struct cpumask *cpus, unsigned long capped_freq) { - unsigned long max_capacity, capacity, hw_pressure; + unsigned long max_capacity, capacity, pressure; u32 max_freq; int cpu; @@ -196,12 +196,12 @@ void topology_update_hw_pressure(const struct cpumask *cpus, else capacity = mult_frac(max_capacity, capped_freq, max_freq); - hw_pressure = max_capacity - capacity; + pressure = max_capacity - capacity; - trace_hw_pressure_update(cpu, hw_pressure); + trace_hw_pressure_update(cpu, pressure); for_each_cpu(cpu, cpus) - WRITE_ONCE(per_cpu(hw_pressure, cpu), hw_pressure); + WRITE_ONCE(per_cpu(hw_pressure, cpu), pressure); } EXPORT_SYMBOL_GPL(topology_update_hw_pressure); diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index 56fba44ba391..c61ecb0c2ae2 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -306,7 +306,7 @@ static ssize_t crash_hotplug_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sysfs_emit(buf, "%d\n", crash_hotplug_cpu_support()); + return sysfs_emit(buf, "%d\n", crash_check_hotplug_support()); } static DEVICE_ATTR_ADMIN_RO(crash_hotplug); #endif diff --git a/drivers/base/memory.c b/drivers/base/memory.c index c0436f46cfb7..67858eeb92ed 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -535,7 +535,7 @@ static DEVICE_ATTR_RW(auto_online_blocks); static ssize_t crash_hotplug_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sysfs_emit(buf, "%d\n", crash_hotplug_memory_support()); + return sysfs_emit(buf, "%d\n", crash_check_hotplug_support()); } static DEVICE_ATTR_RO(crash_hotplug); #endif diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 21728e9ea5c3..8a2ce8070010 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -2215,6 +2215,7 @@ static int pkt_open_dev(struct pktcdvd_device *pd, bool write) } dev_info(ddev, "%lukB available on disc\n", lba << 1); } + set_blocksize(bdev_file, CD_FRAMESIZE); return 0; @@ -2278,11 +2279,6 @@ static int pkt_open(struct gendisk *disk, blk_mode_t mode) ret = pkt_open_dev(pd, mode & BLK_OPEN_WRITE); if (ret) goto out_dec; - /* - * needed here as well, since ext2 (among others) may change - * the blocksize at mount time - */ - set_blocksize(disk->part0, CD_FRAMESIZE); } mutex_unlock(&ctl_mutex); mutex_unlock(&pktcdvd_mutex); @@ -2526,7 +2522,6 @@ static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev) __module_get(THIS_MODULE); pd->bdev_file = bdev_file; - set_blocksize(file_bdev(bdev_file), CD_FRAMESIZE); atomic_set(&pd->cdrw.pending_bios, 0); pd->cdrw.thread = kthread_run(kcdrwd, pd, "%s", pd->disk->disk_name); diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index f0639df6cd18..3acd7006ad2c 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -426,11 +426,10 @@ static void reset_bdev(struct zram *zram) if (!zram->backing_dev) return; - fput(zram->bdev_file); /* hope filp_close flush all of IO */ filp_close(zram->backing_dev, NULL); zram->backing_dev = NULL; - zram->bdev_file = NULL; + zram->bdev = NULL; zram->disk->fops = &zram_devops; kvfree(zram->bitmap); zram->bitmap = NULL; @@ -473,10 +472,8 @@ static ssize_t backing_dev_store(struct device *dev, size_t sz; struct file *backing_dev = NULL; struct inode *inode; - struct address_space *mapping; unsigned int bitmap_sz; unsigned long nr_pages, *bitmap = NULL; - struct file *bdev_file = NULL; int err; struct zram *zram = dev_to_zram(dev); @@ -497,15 +494,14 @@ static ssize_t backing_dev_store(struct device *dev, if (sz > 0 && file_name[sz - 1] == '\n') file_name[sz - 1] = 0x00; - backing_dev = filp_open(file_name, O_RDWR|O_LARGEFILE, 0); + backing_dev = filp_open(file_name, O_RDWR | O_LARGEFILE | O_EXCL, 0); if (IS_ERR(backing_dev)) { err = PTR_ERR(backing_dev); backing_dev = NULL; goto out; } - mapping = backing_dev->f_mapping; - inode = mapping->host; + inode = backing_dev->f_mapping->host; /* Support only block device in this moment */ if (!S_ISBLK(inode->i_mode)) { @@ -513,14 +509,6 @@ static ssize_t backing_dev_store(struct device *dev, goto out; } - bdev_file = bdev_file_open_by_dev(inode->i_rdev, - BLK_OPEN_READ | BLK_OPEN_WRITE, zram, NULL); - if (IS_ERR(bdev_file)) { - err = PTR_ERR(bdev_file); - bdev_file = NULL; - goto out; - } - nr_pages = i_size_read(inode) >> PAGE_SHIFT; bitmap_sz = BITS_TO_LONGS(nr_pages) * sizeof(long); bitmap = kvzalloc(bitmap_sz, GFP_KERNEL); @@ -531,7 +519,7 @@ static ssize_t backing_dev_store(struct device *dev, reset_bdev(zram); - zram->bdev_file = bdev_file; + zram->bdev = I_BDEV(inode); zram->backing_dev = backing_dev; zram->bitmap = bitmap; zram->nr_pages = nr_pages; @@ -544,9 +532,6 @@ static ssize_t backing_dev_store(struct device *dev, out: kvfree(bitmap); - if (bdev_file) - fput(bdev_file); - if (backing_dev) filp_close(backing_dev, NULL); @@ -587,7 +572,7 @@ static void read_from_bdev_async(struct zram *zram, struct page *page, { struct bio *bio; - bio = bio_alloc(file_bdev(zram->bdev_file), 1, parent->bi_opf, GFP_NOIO); + bio = bio_alloc(zram->bdev, 1, parent->bi_opf, GFP_NOIO); bio->bi_iter.bi_sector = entry * (PAGE_SIZE >> 9); __bio_add_page(bio, page, PAGE_SIZE, 0); bio_chain(bio, parent); @@ -703,7 +688,7 @@ static ssize_t writeback_store(struct device *dev, continue; } - bio_init(&bio, file_bdev(zram->bdev_file), &bio_vec, 1, + bio_init(&bio, zram->bdev, &bio_vec, 1, REQ_OP_WRITE | REQ_SYNC); bio.bi_iter.bi_sector = blk_idx * (PAGE_SIZE >> 9); __bio_add_page(&bio, page, PAGE_SIZE, 0); @@ -785,7 +770,7 @@ static void zram_sync_read(struct work_struct *work) struct bio_vec bv; struct bio bio; - bio_init(&bio, file_bdev(zw->zram->bdev_file), &bv, 1, REQ_OP_READ); + bio_init(&bio, zw->zram->bdev, &bv, 1, REQ_OP_READ); bio.bi_iter.bi_sector = zw->entry * (PAGE_SIZE >> 9); __bio_add_page(&bio, zw->page, PAGE_SIZE, 0); zw->error = submit_bio_wait(&bio); @@ -1568,7 +1553,8 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, * Corresponding ZRAM slot should be locked. */ static int zram_recompress(struct zram *zram, u32 index, struct page *page, - u32 threshold, u32 prio, u32 prio_max) + u64 *num_recomp_pages, u32 threshold, u32 prio, + u32 prio_max) { struct zcomp_strm *zstrm = NULL; unsigned long handle_old; @@ -1645,6 +1631,15 @@ static int zram_recompress(struct zram *zram, u32 index, struct page *page, if (!zstrm) return 0; + /* + * Decrement the limit (if set) on pages we can recompress, even + * when current recompression was unsuccessful or did not compress + * the page below the threshold, because we still spent resources + * on it. + */ + if (*num_recomp_pages) + *num_recomp_pages -= 1; + if (class_index_new >= class_index_old) { /* * Secondary algorithms failed to re-compress the page @@ -1710,6 +1705,7 @@ static ssize_t recompress_store(struct device *dev, struct zram *zram = dev_to_zram(dev); unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; char *args, *param, *val, *algo = NULL; + u64 num_recomp_pages = ULLONG_MAX; u32 mode = 0, threshold = 0; unsigned long index; struct page *page; @@ -1732,6 +1728,17 @@ static ssize_t recompress_store(struct device *dev, continue; } + if (!strcmp(param, "max_pages")) { + /* + * Limit the number of entries (pages) we attempt to + * recompress. + */ + ret = kstrtoull(val, 10, &num_recomp_pages); + if (ret) + return ret; + continue; + } + if (!strcmp(param, "threshold")) { /* * We will re-compress only idle objects equal or @@ -1788,6 +1795,9 @@ static ssize_t recompress_store(struct device *dev, for (index = 0; index < nr_pages; index++) { int err = 0; + if (!num_recomp_pages) + break; + zram_slot_lock(zram, index); if (!zram_allocated(zram, index)) @@ -1807,8 +1817,8 @@ static ssize_t recompress_store(struct device *dev, zram_test_flag(zram, index, ZRAM_INCOMPRESSIBLE)) goto next; - err = zram_recompress(zram, index, page, threshold, - prio, prio_max); + err = zram_recompress(zram, index, page, &num_recomp_pages, + threshold, prio, prio_max); next: zram_slot_unlock(zram, index); if (err) { diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 37bf29f34d26..35e322144629 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -132,7 +132,7 @@ struct zram { spinlock_t wb_limit_lock; bool wb_limit_enable; u64 bd_wb_limit; - struct file *bdev_file; + struct block_device *bdev; unsigned long *bitmap; unsigned long nr_pages; #endif diff --git a/drivers/char/hw_random/nomadik-rng.c b/drivers/char/hw_random/nomadik-rng.c index a2009fc4ad3c..f2a2aa7a531c 100644 --- a/drivers/char/hw_random/nomadik-rng.c +++ b/drivers/char/hw_random/nomadik-rng.c @@ -78,7 +78,6 @@ MODULE_DEVICE_TABLE(amba, nmk_rng_ids); static struct amba_driver nmk_rng_driver = { .drv = { - .owner = THIS_MODULE, .name = "rng", }, .probe = nmk_rng_probe, diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 3c6670cf905f..9b80e622ae80 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -544,7 +544,7 @@ static unsigned long get_unmapped_area_zero(struct file *file, } /* Otherwise flags & MAP_PRIVATE: with no shmem object beneath it */ - return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, file, addr, len, pgoff, flags); #else return -ENOSYS; #endif diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 7517a0dfd15c..3e9099504fad 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -489,6 +489,7 @@ source "drivers/clk/rockchip/Kconfig" source "drivers/clk/samsung/Kconfig" source "drivers/clk/sifive/Kconfig" source "drivers/clk/socfpga/Kconfig" +source "drivers/clk/sophgo/Kconfig" source "drivers/clk/sprd/Kconfig" source "drivers/clk/starfive/Kconfig" source "drivers/clk/sunxi/Kconfig" diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index 14fa8d4ecc1f..4abe16c8ccdf 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -118,6 +118,7 @@ obj-$(CONFIG_ARCH_ROCKCHIP) += rockchip/ obj-$(CONFIG_COMMON_CLK_SAMSUNG) += samsung/ obj-$(CONFIG_CLK_SIFIVE) += sifive/ obj-y += socfpga/ +obj-y += sophgo/ obj-$(CONFIG_PLAT_SPEAR) += spear/ obj-y += sprd/ obj-$(CONFIG_ARCH_STI) += st/ diff --git a/drivers/clk/bcm/clk-bcm2711-dvp.c b/drivers/clk/bcm/clk-bcm2711-dvp.c index e4fbbf3c40fe..3cb235df9d37 100644 --- a/drivers/clk/bcm/clk-bcm2711-dvp.c +++ b/drivers/clk/bcm/clk-bcm2711-dvp.c @@ -56,6 +56,8 @@ static int clk_dvp_probe(struct platform_device *pdev) if (ret) return ret; + data->num = NR_CLOCKS; + data->hws[0] = clk_hw_register_gate_parent_data(&pdev->dev, "hdmi0-108MHz", &clk_dvp_parent, 0, @@ -76,7 +78,6 @@ static int clk_dvp_probe(struct platform_device *pdev) goto unregister_clk0; } - data->num = NR_CLOCKS; ret = of_clk_add_hw_provider(pdev->dev.of_node, of_clk_hw_onecell_get, data); if (ret) diff --git a/drivers/clk/bcm/clk-raspberrypi.c b/drivers/clk/bcm/clk-raspberrypi.c index 829406dc44a2..4d411408e4af 100644 --- a/drivers/clk/bcm/clk-raspberrypi.c +++ b/drivers/clk/bcm/clk-raspberrypi.c @@ -371,8 +371,8 @@ static int raspberrypi_discover_clocks(struct raspberrypi_clk *rpi, if (IS_ERR(hw)) return PTR_ERR(hw); - data->hws[clks->id] = hw; data->num = clks->id + 1; + data->hws[clks->id] = hw; } clks++; diff --git a/drivers/clk/clk-en7523.c b/drivers/clk/clk-en7523.c index 7cde328495e2..ccc394692671 100644 --- a/drivers/clk/clk-en7523.c +++ b/drivers/clk/clk-en7523.c @@ -3,14 +3,16 @@ #include <linux/delay.h> #include <linux/clk-provider.h> #include <linux/io.h> -#include <linux/of.h> #include <linux/platform_device.h> +#include <linux/property.h> #include <dt-bindings/clock/en7523-clk.h> #define REG_PCI_CONTROL 0x88 #define REG_PCI_CONTROL_PERSTOUT BIT(29) #define REG_PCI_CONTROL_PERSTOUT1 BIT(26) +#define REG_PCI_CONTROL_REFCLK_EN0 BIT(23) #define REG_PCI_CONTROL_REFCLK_EN1 BIT(22) +#define REG_PCI_CONTROL_PERSTOUT2 BIT(16) #define REG_GSW_CLK_DIV_SEL 0x1b4 #define REG_EMI_CLK_DIV_SEL 0x1b8 #define REG_BUS_CLK_DIV_SEL 0x1bc @@ -18,10 +20,25 @@ #define REG_SPI_CLK_FREQ_SEL 0x1c8 #define REG_NPU_CLK_DIV_SEL 0x1fc #define REG_CRYPTO_CLKSRC 0x200 -#define REG_RESET_CONTROL 0x834 +#define REG_RESET_CONTROL2 0x830 +#define REG_RESET2_CONTROL_PCIE2 BIT(27) +#define REG_RESET_CONTROL1 0x834 #define REG_RESET_CONTROL_PCIEHB BIT(29) #define REG_RESET_CONTROL_PCIE1 BIT(27) #define REG_RESET_CONTROL_PCIE2 BIT(26) +/* EN7581 */ +#define REG_PCIE0_MEM 0x00 +#define REG_PCIE0_MEM_MASK 0x04 +#define REG_PCIE1_MEM 0x08 +#define REG_PCIE1_MEM_MASK 0x0c +#define REG_PCIE2_MEM 0x10 +#define REG_PCIE2_MEM_MASK 0x14 +#define REG_PCIE_RESET_OPEN_DRAIN 0x018c +#define REG_PCIE_RESET_OPEN_DRAIN_MASK GENMASK(2, 0) +#define REG_NP_SCU_PCIC 0x88 +#define REG_NP_SCU_SSTR 0x9c +#define REG_PCIE_XSI0_SEL_MASK GENMASK(14, 13) +#define REG_PCIE_XSI1_SEL_MASK GENMASK(12, 11) struct en_clk_desc { int id; @@ -47,6 +64,12 @@ struct en_clk_gate { struct clk_hw hw; }; +struct en_clk_soc_data { + const struct clk_ops pcie_ops; + int (*hw_init)(struct platform_device *pdev, void __iomem *base, + void __iomem *np_base); +}; + static const u32 gsw_base[] = { 400000000, 500000000 }; static const u32 emi_base[] = { 333000000, 400000000 }; static const u32 bus_base[] = { 500000000, 540000000 }; @@ -145,11 +168,6 @@ static const struct en_clk_desc en7523_base_clks[] = { } }; -static const struct of_device_id of_match_clk_en7523[] = { - { .compatible = "airoha,en7523-scu", }, - { /* sentinel */ } -}; - static unsigned int en7523_get_base_rate(void __iomem *base, unsigned int i) { const struct en_clk_desc *desc = &en7523_base_clks[i]; @@ -212,14 +230,14 @@ static int en7523_pci_prepare(struct clk_hw *hw) usleep_range(1000, 2000); /* Reset to default */ - val = readl(np_base + REG_RESET_CONTROL); + val = readl(np_base + REG_RESET_CONTROL1); mask = REG_RESET_CONTROL_PCIE1 | REG_RESET_CONTROL_PCIE2 | REG_RESET_CONTROL_PCIEHB; - writel(val & ~mask, np_base + REG_RESET_CONTROL); + writel(val & ~mask, np_base + REG_RESET_CONTROL1); usleep_range(1000, 2000); - writel(val | mask, np_base + REG_RESET_CONTROL); + writel(val | mask, np_base + REG_RESET_CONTROL1); msleep(100); - writel(val & ~mask, np_base + REG_RESET_CONTROL); + writel(val & ~mask, np_base + REG_RESET_CONTROL1); usleep_range(5000, 10000); /* Release device */ @@ -247,14 +265,10 @@ static void en7523_pci_unprepare(struct clk_hw *hw) static struct clk_hw *en7523_register_pcie_clk(struct device *dev, void __iomem *np_base) { - static const struct clk_ops pcie_gate_ops = { - .is_enabled = en7523_pci_is_enabled, - .prepare = en7523_pci_prepare, - .unprepare = en7523_pci_unprepare, - }; + const struct en_clk_soc_data *soc_data = device_get_match_data(dev); struct clk_init_data init = { .name = "pcie", - .ops = &pcie_gate_ops, + .ops = &soc_data->pcie_ops, }; struct en_clk_gate *cg; @@ -264,7 +278,10 @@ static struct clk_hw *en7523_register_pcie_clk(struct device *dev, cg->base = np_base; cg->hw.init = &init; - en7523_pci_unprepare(&cg->hw); + + if (init.ops->disable) + init.ops->disable(&cg->hw); + init.ops->unprepare(&cg->hw); if (clk_hw_register(dev, &cg->hw)) return NULL; @@ -272,6 +289,111 @@ static struct clk_hw *en7523_register_pcie_clk(struct device *dev, return &cg->hw; } +static int en7581_pci_is_enabled(struct clk_hw *hw) +{ + struct en_clk_gate *cg = container_of(hw, struct en_clk_gate, hw); + u32 val, mask; + + mask = REG_PCI_CONTROL_REFCLK_EN0 | REG_PCI_CONTROL_REFCLK_EN1; + val = readl(cg->base + REG_PCI_CONTROL); + return (val & mask) == mask; +} + +static int en7581_pci_prepare(struct clk_hw *hw) +{ + struct en_clk_gate *cg = container_of(hw, struct en_clk_gate, hw); + void __iomem *np_base = cg->base; + u32 val, mask; + + mask = REG_RESET_CONTROL_PCIE1 | REG_RESET_CONTROL_PCIE2 | + REG_RESET_CONTROL_PCIEHB; + val = readl(np_base + REG_RESET_CONTROL1); + writel(val & ~mask, np_base + REG_RESET_CONTROL1); + val = readl(np_base + REG_RESET_CONTROL2); + writel(val & ~REG_RESET2_CONTROL_PCIE2, np_base + REG_RESET_CONTROL2); + usleep_range(5000, 10000); + + return 0; +} + +static int en7581_pci_enable(struct clk_hw *hw) +{ + struct en_clk_gate *cg = container_of(hw, struct en_clk_gate, hw); + void __iomem *np_base = cg->base; + u32 val, mask; + + mask = REG_PCI_CONTROL_REFCLK_EN0 | REG_PCI_CONTROL_REFCLK_EN1 | + REG_PCI_CONTROL_PERSTOUT1 | REG_PCI_CONTROL_PERSTOUT2 | + REG_PCI_CONTROL_PERSTOUT; + val = readl(np_base + REG_PCI_CONTROL); + writel(val | mask, np_base + REG_PCI_CONTROL); + msleep(250); + + return 0; +} + +static void en7581_pci_unprepare(struct clk_hw *hw) +{ + struct en_clk_gate *cg = container_of(hw, struct en_clk_gate, hw); + void __iomem *np_base = cg->base; + u32 val, mask; + + mask = REG_RESET_CONTROL_PCIE1 | REG_RESET_CONTROL_PCIE2 | + REG_RESET_CONTROL_PCIEHB; + val = readl(np_base + REG_RESET_CONTROL1); + writel(val | mask, np_base + REG_RESET_CONTROL1); + mask = REG_RESET_CONTROL_PCIE1 | REG_RESET_CONTROL_PCIE2; + writel(val | mask, np_base + REG_RESET_CONTROL1); + val = readl(np_base + REG_RESET_CONTROL2); + writel(val | REG_RESET_CONTROL_PCIE2, np_base + REG_RESET_CONTROL2); + msleep(100); +} + +static void en7581_pci_disable(struct clk_hw *hw) +{ + struct en_clk_gate *cg = container_of(hw, struct en_clk_gate, hw); + void __iomem *np_base = cg->base; + u32 val, mask; + + mask = REG_PCI_CONTROL_REFCLK_EN0 | REG_PCI_CONTROL_REFCLK_EN1 | + REG_PCI_CONTROL_PERSTOUT1 | REG_PCI_CONTROL_PERSTOUT2 | + REG_PCI_CONTROL_PERSTOUT; + val = readl(np_base + REG_PCI_CONTROL); + writel(val & ~mask, np_base + REG_PCI_CONTROL); + usleep_range(1000, 2000); +} + +static int en7581_clk_hw_init(struct platform_device *pdev, + void __iomem *base, + void __iomem *np_base) +{ + void __iomem *pb_base; + u32 val; + + pb_base = devm_platform_ioremap_resource(pdev, 2); + if (IS_ERR(pb_base)) + return PTR_ERR(pb_base); + + val = readl(np_base + REG_NP_SCU_SSTR); + val &= ~(REG_PCIE_XSI0_SEL_MASK | REG_PCIE_XSI1_SEL_MASK); + writel(val, np_base + REG_NP_SCU_SSTR); + val = readl(np_base + REG_NP_SCU_PCIC); + writel(val | 3, np_base + REG_NP_SCU_PCIC); + + writel(0x20000000, pb_base + REG_PCIE0_MEM); + writel(0xfc000000, pb_base + REG_PCIE0_MEM_MASK); + writel(0x24000000, pb_base + REG_PCIE1_MEM); + writel(0xfc000000, pb_base + REG_PCIE1_MEM_MASK); + writel(0x28000000, pb_base + REG_PCIE2_MEM); + writel(0xfc000000, pb_base + REG_PCIE2_MEM_MASK); + + val = readl(base + REG_PCIE_RESET_OPEN_DRAIN); + writel(val | REG_PCIE_RESET_OPEN_DRAIN_MASK, + base + REG_PCIE_RESET_OPEN_DRAIN); + + return 0; +} + static void en7523_register_clocks(struct device *dev, struct clk_hw_onecell_data *clk_data, void __iomem *base, void __iomem *np_base) { @@ -304,6 +426,7 @@ static void en7523_register_clocks(struct device *dev, struct clk_hw_onecell_dat static int en7523_clk_probe(struct platform_device *pdev) { struct device_node *node = pdev->dev.of_node; + const struct en_clk_soc_data *soc_data; struct clk_hw_onecell_data *clk_data; void __iomem *base, *np_base; int r; @@ -316,6 +439,13 @@ static int en7523_clk_probe(struct platform_device *pdev) if (IS_ERR(np_base)) return PTR_ERR(np_base); + soc_data = device_get_match_data(&pdev->dev); + if (soc_data->hw_init) { + r = soc_data->hw_init(pdev, base, np_base); + if (r) + return r; + } + clk_data = devm_kzalloc(&pdev->dev, struct_size(clk_data, hws, EN7523_NUM_CLOCKS), GFP_KERNEL); @@ -333,6 +463,31 @@ static int en7523_clk_probe(struct platform_device *pdev) return r; } +static const struct en_clk_soc_data en7523_data = { + .pcie_ops = { + .is_enabled = en7523_pci_is_enabled, + .prepare = en7523_pci_prepare, + .unprepare = en7523_pci_unprepare, + }, +}; + +static const struct en_clk_soc_data en7581_data = { + .pcie_ops = { + .is_enabled = en7581_pci_is_enabled, + .prepare = en7581_pci_prepare, + .enable = en7581_pci_enable, + .unprepare = en7581_pci_unprepare, + .disable = en7581_pci_disable, + }, + .hw_init = en7581_clk_hw_init, +}; + +static const struct of_device_id of_match_clk_en7523[] = { + { .compatible = "airoha,en7523-scu", .data = &en7523_data }, + { .compatible = "airoha,en7581-scu", .data = &en7581_data }, + { /* sentinel */ } +}; + static struct platform_driver clk_en7523_drv = { .probe = en7523_clk_probe, .driver = { diff --git a/drivers/clk/clk-gemini.c b/drivers/clk/clk-gemini.c index ba0ff01bf4dc..856b008e07c6 100644 --- a/drivers/clk/clk-gemini.c +++ b/drivers/clk/clk-gemini.c @@ -67,12 +67,10 @@ struct gemini_gate_data { * struct clk_gemini_pci - Gemini PCI clock * @hw: corresponding clock hardware entry * @map: regmap to access the registers - * @rate: current rate */ struct clk_gemini_pci { struct clk_hw hw; struct regmap *map; - unsigned long rate; }; /** diff --git a/drivers/clk/clk-highbank.c b/drivers/clk/clk-highbank.c index 2a0cea2946f9..6e68a41a70a1 100644 --- a/drivers/clk/clk-highbank.c +++ b/drivers/clk/clk-highbank.c @@ -37,7 +37,6 @@ struct hb_clk { struct clk_hw hw; void __iomem *reg; - char *parent_name; }; #define to_hb_clk(p) container_of(p, struct hb_clk, hw) diff --git a/drivers/clk/clk-loongson2.c b/drivers/clk/clk-loongson2.c index bacdcbb287ac..820bb1e9e3b7 100644 --- a/drivers/clk/clk-loongson2.c +++ b/drivers/clk/clk-loongson2.c @@ -13,317 +13,348 @@ #include <linux/io-64-nonatomic-lo-hi.h> #include <dt-bindings/clock/loongson,ls2k-clk.h> -#define LOONGSON2_PLL_MULT_SHIFT 32 -#define LOONGSON2_PLL_MULT_WIDTH 10 -#define LOONGSON2_PLL_DIV_SHIFT 26 -#define LOONGSON2_PLL_DIV_WIDTH 6 -#define LOONGSON2_APB_FREQSCALE_SHIFT 20 -#define LOONGSON2_APB_FREQSCALE_WIDTH 3 -#define LOONGSON2_USB_FREQSCALE_SHIFT 16 -#define LOONGSON2_USB_FREQSCALE_WIDTH 3 -#define LOONGSON2_SATA_FREQSCALE_SHIFT 12 -#define LOONGSON2_SATA_FREQSCALE_WIDTH 3 -#define LOONGSON2_BOOT_FREQSCALE_SHIFT 8 -#define LOONGSON2_BOOT_FREQSCALE_WIDTH 3 - -static void __iomem *loongson2_pll_base; - static const struct clk_parent_data pdata[] = { - { .fw_name = "ref_100m",}, + { .fw_name = "ref_100m", }, }; -static struct clk_hw *loongson2_clk_register(struct device *dev, - const char *name, - const char *parent_name, - const struct clk_ops *ops, - unsigned long flags) -{ - int ret; - struct clk_hw *hw; - struct clk_init_data init = { }; - - hw = devm_kzalloc(dev, sizeof(*hw), GFP_KERNEL); - if (!hw) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = ops; - init.flags = flags; - init.num_parents = 1; - - if (!parent_name) - init.parent_data = pdata; - else - init.parent_names = &parent_name; - - hw->init = &init; - - ret = devm_clk_hw_register(dev, hw); - if (ret) - hw = ERR_PTR(ret); - - return hw; -} - -static unsigned long loongson2_calc_pll_rate(int offset, unsigned long rate) -{ - u64 val; - u32 mult, div; - - val = readq(loongson2_pll_base + offset); - - mult = (val >> LOONGSON2_PLL_MULT_SHIFT) & - clk_div_mask(LOONGSON2_PLL_MULT_WIDTH); - div = (val >> LOONGSON2_PLL_DIV_SHIFT) & - clk_div_mask(LOONGSON2_PLL_DIV_WIDTH); - - return div_u64((u64)rate * mult, div); -} +enum loongson2_clk_type { + CLK_TYPE_PLL, + CLK_TYPE_SCALE, + CLK_TYPE_DIVIDER, + CLK_TYPE_GATE, + CLK_TYPE_FIXED, + CLK_TYPE_NONE, +}; -static unsigned long loongson2_node_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return loongson2_calc_pll_rate(0x0, parent_rate); -} +struct loongson2_clk_provider { + void __iomem *base; + struct device *dev; + struct clk_hw_onecell_data clk_data; + spinlock_t clk_lock; /* protect access to DIV registers */ +}; -static const struct clk_ops loongson2_node_clk_ops = { - .recalc_rate = loongson2_node_recalc_rate, +struct loongson2_clk_data { + struct clk_hw hw; + void __iomem *reg; + u8 div_shift; + u8 div_width; + u8 mult_shift; + u8 mult_width; }; -static unsigned long loongson2_ddr_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return loongson2_calc_pll_rate(0x10, parent_rate); -} +struct loongson2_clk_board_info { + u8 id; + enum loongson2_clk_type type; + const char *name; + const char *parent_name; + unsigned long fixed_rate; + u8 reg_offset; + u8 div_shift; + u8 div_width; + u8 mult_shift; + u8 mult_width; + u8 bit_idx; +}; -static const struct clk_ops loongson2_ddr_clk_ops = { - .recalc_rate = loongson2_ddr_recalc_rate, +#define CLK_DIV(_id, _name, _pname, _offset, _dshift, _dwidth) \ + { \ + .id = _id, \ + .type = CLK_TYPE_DIVIDER, \ + .name = _name, \ + .parent_name = _pname, \ + .reg_offset = _offset, \ + .div_shift = _dshift, \ + .div_width = _dwidth, \ + } + +#define CLK_PLL(_id, _name, _offset, _mshift, _mwidth, \ + _dshift, _dwidth) \ + { \ + .id = _id, \ + .type = CLK_TYPE_PLL, \ + .name = _name, \ + .parent_name = NULL, \ + .reg_offset = _offset, \ + .mult_shift = _mshift, \ + .mult_width = _mwidth, \ + .div_shift = _dshift, \ + .div_width = _dwidth, \ + } + +#define CLK_SCALE(_id, _name, _pname, _offset, \ + _dshift, _dwidth) \ + { \ + .id = _id, \ + .type = CLK_TYPE_SCALE, \ + .name = _name, \ + .parent_name = _pname, \ + .reg_offset = _offset, \ + .div_shift = _dshift, \ + .div_width = _dwidth, \ + } + +#define CLK_GATE(_id, _name, _pname, _offset, _bidx) \ + { \ + .id = _id, \ + .type = CLK_TYPE_GATE, \ + .name = _name, \ + .parent_name = _pname, \ + .reg_offset = _offset, \ + .bit_idx = _bidx, \ + } + +#define CLK_FIXED(_id, _name, _pname, _rate) \ + { \ + .id = _id, \ + .type = CLK_TYPE_FIXED, \ + .name = _name, \ + .parent_name = _pname, \ + .fixed_rate = _rate, \ + } + +static const struct loongson2_clk_board_info ls2k0500_clks[] = { + CLK_PLL(LOONGSON2_NODE_PLL, "pll_node", 0, 16, 8, 8, 6), + CLK_PLL(LOONGSON2_DDR_PLL, "pll_ddr", 0x8, 16, 8, 8, 6), + CLK_PLL(LOONGSON2_DC_PLL, "pll_soc", 0x10, 16, 8, 8, 6), + CLK_PLL(LOONGSON2_PIX0_PLL, "pll_pix0", 0x18, 16, 8, 8, 6), + CLK_PLL(LOONGSON2_PIX1_PLL, "pll_pix1", 0x20, 16, 8, 8, 6), + CLK_DIV(LOONGSON2_NODE_CLK, "clk_node", "pll_node", 0, 24, 6), + CLK_DIV(LOONGSON2_DDR_CLK, "clk_ddr", "pll_ddr", 0x8, 24, 6), + CLK_DIV(LOONGSON2_HDA_CLK, "clk_hda", "pll_ddr", 0xc, 8, 6), + CLK_DIV(LOONGSON2_GPU_CLK, "clk_gpu", "pll_soc", 0x10, 24, 6), + CLK_DIV(LOONGSON2_DC_CLK, "clk_sb", "pll_soc", 0x14, 0, 6), + CLK_DIV(LOONGSON2_GMAC_CLK, "clk_gmac", "pll_soc", 0x14, 8, 6), + CLK_DIV(LOONGSON2_PIX0_CLK, "clk_pix0", "pll_pix0", 0x18, 24, 6), + CLK_DIV(LOONGSON2_PIX1_CLK, "clk_pix1", "pll_pix1", 0x20, 24, 6), + CLK_SCALE(LOONGSON2_BOOT_CLK, "clk_boot", "clk_sb", 0x28, 8, 3), + CLK_SCALE(LOONGSON2_SATA_CLK, "clk_sata", "clk_sb", 0x28, 12, 3), + CLK_SCALE(LOONGSON2_USB_CLK, "clk_usb", "clk_sb", 0x28, 16, 3), + CLK_SCALE(LOONGSON2_APB_CLK, "clk_apb", "clk_sb", 0x28, 20, 3), + { /* Sentinel */ }, }; -static unsigned long loongson2_dc_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return loongson2_calc_pll_rate(0x20, parent_rate); -} +static const struct loongson2_clk_board_info ls2k1000_clks[] = { + CLK_PLL(LOONGSON2_NODE_PLL, "pll_node", 0, 32, 10, 26, 6), + CLK_PLL(LOONGSON2_DDR_PLL, "pll_ddr", 0x10, 32, 10, 26, 6), + CLK_PLL(LOONGSON2_DC_PLL, "pll_dc", 0x20, 32, 10, 26, 6), + CLK_PLL(LOONGSON2_PIX0_PLL, "pll_pix0", 0x30, 32, 10, 26, 6), + CLK_PLL(LOONGSON2_PIX1_PLL, "pll_pix1", 0x40, 32, 10, 26, 6), + CLK_DIV(LOONGSON2_NODE_CLK, "clk_node", "pll_node", 0x8, 0, 6), + CLK_DIV(LOONGSON2_DDR_CLK, "clk_ddr", "pll_ddr", 0x18, 0, 6), + CLK_DIV(LOONGSON2_GPU_CLK, "clk_gpu", "pll_ddr", 0x18, 22, 6), + /* + * The hda clk divisor in the upper 32bits and the clk-prodiver + * layer code doesn't support 64bit io operation thus a conversion + * is required that subtract shift by 32 and add 4byte to the hda + * address + */ + CLK_DIV(LOONGSON2_HDA_CLK, "clk_hda", "pll_ddr", 0x22, 12, 7), + CLK_DIV(LOONGSON2_DC_CLK, "clk_dc", "pll_dc", 0x28, 0, 6), + CLK_DIV(LOONGSON2_GMAC_CLK, "clk_gmac", "pll_dc", 0x28, 22, 6), + CLK_DIV(LOONGSON2_PIX0_CLK, "clk_pix0", "pll_pix0", 0x38, 0, 6), + CLK_DIV(LOONGSON2_PIX1_CLK, "clk_pix1", "pll_pix1", 0x38, 0, 6), + CLK_SCALE(LOONGSON2_BOOT_CLK, "clk_boot", NULL, 0x50, 8, 3), + CLK_SCALE(LOONGSON2_SATA_CLK, "clk_sata", "clk_gmac", 0x50, 12, 3), + CLK_SCALE(LOONGSON2_USB_CLK, "clk_usb", "clk_gmac", 0x50, 16, 3), + CLK_SCALE(LOONGSON2_APB_CLK, "clk_apb", "clk_gmac", 0x50, 20, 3), + { /* Sentinel */ }, +}; -static const struct clk_ops loongson2_dc_clk_ops = { - .recalc_rate = loongson2_dc_recalc_rate, +static const struct loongson2_clk_board_info ls2k2000_clks[] = { + CLK_PLL(LOONGSON2_DC_PLL, "pll_0", 0, 21, 9, 32, 6), + CLK_PLL(LOONGSON2_DDR_PLL, "pll_1", 0x10, 21, 9, 32, 6), + CLK_PLL(LOONGSON2_NODE_PLL, "pll_2", 0x20, 21, 9, 32, 6), + CLK_PLL(LOONGSON2_PIX0_PLL, "pll_pix0", 0x30, 21, 9, 32, 6), + CLK_PLL(LOONGSON2_PIX1_PLL, "pll_pix1", 0x40, 21, 9, 32, 6), + CLK_GATE(LOONGSON2_OUT0_GATE, "out0_gate", "pll_0", 0, 40), + CLK_GATE(LOONGSON2_GMAC_GATE, "gmac_gate", "pll_0", 0, 41), + CLK_GATE(LOONGSON2_RIO_GATE, "rio_gate", "pll_0", 0, 42), + CLK_GATE(LOONGSON2_DC_GATE, "dc_gate", "pll_1", 0x10, 40), + CLK_GATE(LOONGSON2_DDR_GATE, "ddr_gate", "pll_1", 0x10, 41), + CLK_GATE(LOONGSON2_GPU_GATE, "gpu_gate", "pll_1", 0x10, 42), + CLK_GATE(LOONGSON2_HDA_GATE, "hda_gate", "pll_2", 0x20, 40), + CLK_GATE(LOONGSON2_NODE_GATE, "node_gate", "pll_2", 0x20, 41), + CLK_GATE(LOONGSON2_EMMC_GATE, "emmc_gate", "pll_2", 0x20, 42), + CLK_GATE(LOONGSON2_PIX0_GATE, "pix0_gate", "pll_pix0", 0x30, 40), + CLK_GATE(LOONGSON2_PIX1_GATE, "pix1_gate", "pll_pix1", 0x40, 40), + CLK_DIV(LOONGSON2_OUT0_CLK, "clk_out0", "out0_gate", 0, 0, 6), + CLK_DIV(LOONGSON2_GMAC_CLK, "clk_gmac", "gmac_gate", 0, 7, 6), + CLK_DIV(LOONGSON2_RIO_CLK, "clk_rio", "rio_gate", 0, 14, 6), + CLK_DIV(LOONGSON2_DC_CLK, "clk_dc", "dc_gate", 0x10, 0, 6), + CLK_DIV(LOONGSON2_GPU_CLK, "clk_gpu", "gpu_gate", 0x10, 7, 6), + CLK_DIV(LOONGSON2_DDR_CLK, "clk_ddr", "ddr_gate", 0x10, 14, 6), + CLK_DIV(LOONGSON2_HDA_CLK, "clk_hda", "hda_gate", 0x20, 0, 6), + CLK_DIV(LOONGSON2_NODE_CLK, "clk_node", "node_gate", 0x20, 7, 6), + CLK_DIV(LOONGSON2_EMMC_CLK, "clk_emmc", "emmc_gate", 0x20, 14, 6), + CLK_DIV(LOONGSON2_PIX0_CLK, "clk_pix0", "pll_pix0", 0x30, 0, 6), + CLK_DIV(LOONGSON2_PIX1_CLK, "clk_pix1", "pll_pix1", 0x40, 0, 6), + CLK_SCALE(LOONGSON2_SATA_CLK, "clk_sata", "clk_out0", 0x50, 12, 3), + CLK_SCALE(LOONGSON2_USB_CLK, "clk_usb", "clk_out0", 0x50, 16, 3), + CLK_SCALE(LOONGSON2_APB_CLK, "clk_apb", "clk_node", 0x50, 20, 3), + CLK_SCALE(LOONGSON2_BOOT_CLK, "clk_boot", NULL, 0x50, 23, 3), + CLK_SCALE(LOONGSON2_DES_CLK, "clk_des", "clk_node", 0x50, 40, 3), + CLK_SCALE(LOONGSON2_I2S_CLK, "clk_i2s", "clk_node", 0x50, 44, 3), + CLK_FIXED(LOONGSON2_MISC_CLK, "clk_misc", NULL, 50000000), + { /* Sentinel */ }, }; -static unsigned long loongson2_pix0_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) +static inline struct loongson2_clk_data *to_loongson2_clk(struct clk_hw *hw) { - return loongson2_calc_pll_rate(0x30, parent_rate); + return container_of(hw, struct loongson2_clk_data, hw); } -static const struct clk_ops loongson2_pix0_clk_ops = { - .recalc_rate = loongson2_pix0_recalc_rate, -}; - -static unsigned long loongson2_pix1_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) +static inline unsigned long loongson2_rate_part(u64 val, u8 shift, u8 width) { - return loongson2_calc_pll_rate(0x40, parent_rate); + return (val & GENMASK(shift + width - 1, shift)) >> shift; } -static const struct clk_ops loongson2_pix1_clk_ops = { - .recalc_rate = loongson2_pix1_recalc_rate, -}; - -static unsigned long loongson2_calc_rate(unsigned long rate, - int shift, int width) +static unsigned long loongson2_pll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) { - u64 val; - u32 mult; - - val = readq(loongson2_pll_base + 0x50); + u64 val, mult, div; + struct loongson2_clk_data *clk = to_loongson2_clk(hw); - mult = (val >> shift) & clk_div_mask(width); + val = readq(clk->reg); + mult = loongson2_rate_part(val, clk->mult_shift, clk->mult_width); + div = loongson2_rate_part(val, clk->div_shift, clk->div_width); - return div_u64((u64)rate * (mult + 1), 8); -} - -static unsigned long loongson2_boot_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return loongson2_calc_rate(parent_rate, - LOONGSON2_BOOT_FREQSCALE_SHIFT, - LOONGSON2_BOOT_FREQSCALE_WIDTH); + return div_u64((u64)parent_rate * mult, div); } -static const struct clk_ops loongson2_boot_clk_ops = { - .recalc_rate = loongson2_boot_recalc_rate, +static const struct clk_ops loongson2_pll_recalc_ops = { + .recalc_rate = loongson2_pll_recalc_rate, }; -static unsigned long loongson2_apb_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) +static unsigned long loongson2_freqscale_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) { - return loongson2_calc_rate(parent_rate, - LOONGSON2_APB_FREQSCALE_SHIFT, - LOONGSON2_APB_FREQSCALE_WIDTH); -} + u64 val, mult; + struct loongson2_clk_data *clk = to_loongson2_clk(hw); -static const struct clk_ops loongson2_apb_clk_ops = { - .recalc_rate = loongson2_apb_recalc_rate, -}; + val = readq(clk->reg); + mult = loongson2_rate_part(val, clk->div_shift, clk->div_width) + 1; -static unsigned long loongson2_usb_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return loongson2_calc_rate(parent_rate, - LOONGSON2_USB_FREQSCALE_SHIFT, - LOONGSON2_USB_FREQSCALE_WIDTH); + return div_u64((u64)parent_rate * mult, 8); } -static const struct clk_ops loongson2_usb_clk_ops = { - .recalc_rate = loongson2_usb_recalc_rate, +static const struct clk_ops loongson2_freqscale_recalc_ops = { + .recalc_rate = loongson2_freqscale_recalc_rate, }; -static unsigned long loongson2_sata_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) +static struct clk_hw *loongson2_clk_register(struct loongson2_clk_provider *clp, + const struct loongson2_clk_board_info *cld, + const struct clk_ops *ops) { - return loongson2_calc_rate(parent_rate, - LOONGSON2_SATA_FREQSCALE_SHIFT, - LOONGSON2_SATA_FREQSCALE_WIDTH); -} + int ret; + struct clk_hw *hw; + struct loongson2_clk_data *clk; + struct clk_init_data init = { }; -static const struct clk_ops loongson2_sata_clk_ops = { - .recalc_rate = loongson2_sata_recalc_rate, -}; + clk = devm_kzalloc(clp->dev, sizeof(*clk), GFP_KERNEL); + if (!clk) + return ERR_PTR(-ENOMEM); -static inline int loongson2_check_clk_hws(struct clk_hw *clks[], unsigned int count) -{ - unsigned int i; + init.name = cld->name; + init.ops = ops; + init.flags = 0; + init.num_parents = 1; - for (i = 0; i < count; i++) - if (IS_ERR(clks[i])) { - pr_err("Loongson2 clk %u: register failed with %ld\n", - i, PTR_ERR(clks[i])); - return PTR_ERR(clks[i]); - } + if (!cld->parent_name) + init.parent_data = pdata; + else + init.parent_names = &cld->parent_name; + + clk->reg = clp->base + cld->reg_offset; + clk->div_shift = cld->div_shift; + clk->div_width = cld->div_width; + clk->mult_shift = cld->mult_shift; + clk->mult_width = cld->mult_width; + clk->hw.init = &init; - return 0; + hw = &clk->hw; + ret = devm_clk_hw_register(clp->dev, hw); + if (ret) + clk = ERR_PTR(ret); + + return hw; } static int loongson2_clk_probe(struct platform_device *pdev) { - int ret; - struct clk_hw **hws; - struct clk_hw_onecell_data *clk_hw_data; - spinlock_t loongson2_clk_lock; + int i, clks_num = 0; + struct clk_hw *hw; struct device *dev = &pdev->dev; + struct loongson2_clk_provider *clp; + const struct loongson2_clk_board_info *p, *data; - loongson2_pll_base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(loongson2_pll_base)) - return PTR_ERR(loongson2_pll_base); - - clk_hw_data = devm_kzalloc(dev, struct_size(clk_hw_data, hws, LOONGSON2_CLK_END), - GFP_KERNEL); - if (WARN_ON(!clk_hw_data)) - return -ENOMEM; - - clk_hw_data->num = LOONGSON2_CLK_END; - hws = clk_hw_data->hws; - - hws[LOONGSON2_NODE_PLL] = loongson2_clk_register(dev, "node_pll", - NULL, - &loongson2_node_clk_ops, 0); - - hws[LOONGSON2_DDR_PLL] = loongson2_clk_register(dev, "ddr_pll", - NULL, - &loongson2_ddr_clk_ops, 0); + data = device_get_match_data(dev); + if (!data) + return -EINVAL; - hws[LOONGSON2_DC_PLL] = loongson2_clk_register(dev, "dc_pll", - NULL, - &loongson2_dc_clk_ops, 0); + for (p = data; p->name; p++) + clks_num++; - hws[LOONGSON2_PIX0_PLL] = loongson2_clk_register(dev, "pix0_pll", - NULL, - &loongson2_pix0_clk_ops, 0); - - hws[LOONGSON2_PIX1_PLL] = loongson2_clk_register(dev, "pix1_pll", - NULL, - &loongson2_pix1_clk_ops, 0); + clp = devm_kzalloc(dev, struct_size(clp, clk_data.hws, clks_num), + GFP_KERNEL); + if (!clp) + return -ENOMEM; - hws[LOONGSON2_BOOT_CLK] = loongson2_clk_register(dev, "boot", - NULL, - &loongson2_boot_clk_ops, 0); + clp->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(clp->base)) + return PTR_ERR(clp->base); + + spin_lock_init(&clp->clk_lock); + clp->clk_data.num = clks_num + 1; + clp->dev = dev; + + for (i = 0; i < clks_num; i++) { + p = &data[i]; + switch (p->type) { + case CLK_TYPE_PLL: + hw = loongson2_clk_register(clp, p, + &loongson2_pll_recalc_ops); + break; + case CLK_TYPE_SCALE: + hw = loongson2_clk_register(clp, p, + &loongson2_freqscale_recalc_ops); + break; + case CLK_TYPE_DIVIDER: + hw = devm_clk_hw_register_divider(dev, p->name, + p->parent_name, 0, + clp->base + p->reg_offset, + p->div_shift, p->div_width, + CLK_DIVIDER_ONE_BASED, + &clp->clk_lock); + break; + case CLK_TYPE_GATE: + hw = devm_clk_hw_register_gate(dev, p->name, p->parent_name, 0, + clp->base + p->reg_offset, + p->bit_idx, 0, + &clp->clk_lock); + break; + case CLK_TYPE_FIXED: + hw = clk_hw_register_fixed_rate_parent_data(dev, p->name, pdata, + 0, p->fixed_rate); + break; + default: + return dev_err_probe(dev, -EINVAL, "Invalid clk type\n"); + } - hws[LOONGSON2_NODE_CLK] = devm_clk_hw_register_divider(dev, "node", - "node_pll", 0, - loongson2_pll_base + 0x8, 0, - 6, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); + if (IS_ERR(hw)) + return dev_err_probe(dev, PTR_ERR(hw), + "Register clk: %s, type: %u failed!\n", + p->name, p->type); - /* - * The hda clk divisor in the upper 32bits and the clk-prodiver - * layer code doesn't support 64bit io operation thus a conversion - * is required that subtract shift by 32 and add 4byte to the hda - * address - */ - hws[LOONGSON2_HDA_CLK] = devm_clk_hw_register_divider(dev, "hda", - "ddr_pll", 0, - loongson2_pll_base + 0x22, 12, - 7, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_GPU_CLK] = devm_clk_hw_register_divider(dev, "gpu", - "ddr_pll", 0, - loongson2_pll_base + 0x18, 22, - 6, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_DDR_CLK] = devm_clk_hw_register_divider(dev, "ddr", - "ddr_pll", 0, - loongson2_pll_base + 0x18, 0, - 6, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_GMAC_CLK] = devm_clk_hw_register_divider(dev, "gmac", - "dc_pll", 0, - loongson2_pll_base + 0x28, 22, - 6, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_DC_CLK] = devm_clk_hw_register_divider(dev, "dc", - "dc_pll", 0, - loongson2_pll_base + 0x28, 0, - 6, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_APB_CLK] = loongson2_clk_register(dev, "apb", - "gmac", - &loongson2_apb_clk_ops, 0); - - hws[LOONGSON2_USB_CLK] = loongson2_clk_register(dev, "usb", - "gmac", - &loongson2_usb_clk_ops, 0); - - hws[LOONGSON2_SATA_CLK] = loongson2_clk_register(dev, "sata", - "gmac", - &loongson2_sata_clk_ops, 0); - - hws[LOONGSON2_PIX0_CLK] = clk_hw_register_divider(NULL, "pix0", - "pix0_pll", 0, - loongson2_pll_base + 0x38, 0, 6, - CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_PIX1_CLK] = clk_hw_register_divider(NULL, "pix1", - "pix1_pll", 0, - loongson2_pll_base + 0x48, 0, 6, - CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - ret = loongson2_check_clk_hws(hws, LOONGSON2_CLK_END); - if (ret) - return ret; + clp->clk_data.hws[p->id] = hw; + } - return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_hw_data); + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, &clp->clk_data); } static const struct of_device_id loongson2_clk_match_table[] = { - { .compatible = "loongson,ls2k-clk" }, + { .compatible = "loongson,ls2k0500-clk", .data = &ls2k0500_clks }, + { .compatible = "loongson,ls2k-clk", .data = &ls2k1000_clks }, + { .compatible = "loongson,ls2k2000-clk", .data = &ls2k2000_clks }, { } }; MODULE_DEVICE_TABLE(of, loongson2_clk_match_table); @@ -338,4 +369,5 @@ static struct platform_driver loongson2_clk_driver = { module_platform_driver(loongson2_clk_driver); MODULE_DESCRIPTION("Loongson2 clock driver"); +MODULE_AUTHOR("Loongson Technology Corporation Limited"); MODULE_LICENSE("GPL"); diff --git a/drivers/clk/clk-renesas-pcie.c b/drivers/clk/clk-renesas-pcie.c index 53e21ac302e6..4c3a5e4eb77a 100644 --- a/drivers/clk/clk-renesas-pcie.c +++ b/drivers/clk/clk-renesas-pcie.c @@ -25,10 +25,12 @@ #define RS9_REG_SS_AMP_0V7 0x1 #define RS9_REG_SS_AMP_0V8 0x2 #define RS9_REG_SS_AMP_0V9 0x3 +#define RS9_REG_SS_AMP_DEFAULT RS9_REG_SS_AMP_0V8 #define RS9_REG_SS_AMP_MASK 0x3 #define RS9_REG_SS_SSC_100 0 #define RS9_REG_SS_SSC_M025 (1 << 3) #define RS9_REG_SS_SSC_M050 (3 << 3) +#define RS9_REG_SS_SSC_DEFAULT RS9_REG_SS_SSC_100 #define RS9_REG_SS_SSC_MASK (3 << 3) #define RS9_REG_SS_SSC_LOCK BIT(5) #define RS9_REG_SR 0x2 @@ -205,8 +207,8 @@ static int rs9_get_common_config(struct rs9_driver_data *rs9) int ret; /* Set defaults */ - rs9->pll_amplitude = RS9_REG_SS_AMP_0V7; - rs9->pll_ssc = RS9_REG_SS_SSC_100; + rs9->pll_amplitude = RS9_REG_SS_AMP_DEFAULT; + rs9->pll_ssc = RS9_REG_SS_SSC_DEFAULT; /* Output clock amplitude */ ret = of_property_read_u32(np, "renesas,out-amplitude-microvolt", @@ -247,13 +249,13 @@ static void rs9_update_config(struct rs9_driver_data *rs9) int i; /* If amplitude is non-default, update it. */ - if (rs9->pll_amplitude != RS9_REG_SS_AMP_0V7) { + if (rs9->pll_amplitude != RS9_REG_SS_AMP_DEFAULT) { regmap_update_bits(rs9->regmap, RS9_REG_SS, RS9_REG_SS_AMP_MASK, rs9->pll_amplitude); } /* If SSC is non-default, update it. */ - if (rs9->pll_ssc != RS9_REG_SS_SSC_100) { + if (rs9->pll_ssc != RS9_REG_SS_SSC_DEFAULT) { regmap_update_bits(rs9->regmap, RS9_REG_SS, RS9_REG_SS_SSC_MASK, rs9->pll_ssc); } diff --git a/drivers/clk/clk-scmi.c b/drivers/clk/clk-scmi.c index 8cbe24789c24..d86a02563f6c 100644 --- a/drivers/clk/clk-scmi.c +++ b/drivers/clk/clk-scmi.c @@ -2,9 +2,10 @@ /* * System Control and Power Interface (SCMI) Protocol based clock driver * - * Copyright (C) 2018-2022 ARM Ltd. + * Copyright (C) 2018-2024 ARM Ltd. */ +#include <linux/bits.h> #include <linux/clk-provider.h> #include <linux/device.h> #include <linux/err.h> @@ -16,6 +17,17 @@ #define NOT_ATOMIC false #define ATOMIC true +enum scmi_clk_feats { + SCMI_CLK_ATOMIC_SUPPORTED, + SCMI_CLK_STATE_CTRL_SUPPORTED, + SCMI_CLK_RATE_CTRL_SUPPORTED, + SCMI_CLK_PARENT_CTRL_SUPPORTED, + SCMI_CLK_DUTY_CYCLE_SUPPORTED, + SCMI_CLK_FEATS_COUNT +}; + +#define SCMI_MAX_CLK_OPS BIT(SCMI_CLK_FEATS_COUNT) + static const struct scmi_clk_proto_ops *scmi_proto_clk_ops; struct scmi_clk { @@ -158,41 +170,44 @@ static int scmi_clk_atomic_is_enabled(struct clk_hw *hw) return !!enabled; } -/* - * We can provide enable/disable/is_enabled atomic callbacks only if the - * underlying SCMI transport for an SCMI instance is configured to handle - * SCMI commands in an atomic manner. - * - * When no SCMI atomic transport support is available we instead provide only - * the prepare/unprepare API, as allowed by the clock framework when atomic - * calls are not available. - * - * Two distinct sets of clk_ops are provided since we could have multiple SCMI - * instances with different underlying transport quality, so they cannot be - * shared. - */ -static const struct clk_ops scmi_clk_ops = { - .recalc_rate = scmi_clk_recalc_rate, - .round_rate = scmi_clk_round_rate, - .set_rate = scmi_clk_set_rate, - .prepare = scmi_clk_enable, - .unprepare = scmi_clk_disable, - .set_parent = scmi_clk_set_parent, - .get_parent = scmi_clk_get_parent, - .determine_rate = scmi_clk_determine_rate, -}; +static int scmi_clk_get_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) +{ + int ret; + u32 val; + struct scmi_clk *clk = to_scmi_clk(hw); -static const struct clk_ops scmi_atomic_clk_ops = { - .recalc_rate = scmi_clk_recalc_rate, - .round_rate = scmi_clk_round_rate, - .set_rate = scmi_clk_set_rate, - .enable = scmi_clk_atomic_enable, - .disable = scmi_clk_atomic_disable, - .is_enabled = scmi_clk_atomic_is_enabled, - .set_parent = scmi_clk_set_parent, - .get_parent = scmi_clk_get_parent, - .determine_rate = scmi_clk_determine_rate, -}; + ret = scmi_proto_clk_ops->config_oem_get(clk->ph, clk->id, + SCMI_CLOCK_CFG_DUTY_CYCLE, + &val, NULL, false); + if (!ret) { + duty->num = val; + duty->den = 100; + } else { + dev_warn(clk->dev, + "Failed to get duty cycle for clock ID %d\n", clk->id); + } + + return ret; +} + +static int scmi_clk_set_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) +{ + int ret; + u32 val; + struct scmi_clk *clk = to_scmi_clk(hw); + + /* SCMI OEM Duty Cycle is expressed as a percentage */ + val = (duty->num * 100) / duty->den; + ret = scmi_proto_clk_ops->config_oem_set(clk->ph, clk->id, + SCMI_CLOCK_CFG_DUTY_CYCLE, + val, false); + if (ret) + dev_warn(clk->dev, + "Failed to set duty cycle(%u/%u) for clock ID %d\n", + duty->num, duty->den, clk->id); + + return ret; +} static int scmi_clk_ops_init(struct device *dev, struct scmi_clk *sclk, const struct clk_ops *scmi_ops) @@ -230,17 +245,153 @@ static int scmi_clk_ops_init(struct device *dev, struct scmi_clk *sclk, return ret; } +/** + * scmi_clk_ops_alloc() - Alloc and configure clock operations + * @dev: A device reference for devres + * @feats_key: A bitmap representing the desired clk_ops capabilities + * + * Allocate and configure a proper set of clock operations depending on the + * specifically required SCMI clock features. + * + * Return: A pointer to the allocated and configured clk_ops on success, + * or NULL on allocation failure. + */ +static const struct clk_ops * +scmi_clk_ops_alloc(struct device *dev, unsigned long feats_key) +{ + struct clk_ops *ops; + + ops = devm_kzalloc(dev, sizeof(*ops), GFP_KERNEL); + if (!ops) + return NULL; + /* + * We can provide enable/disable/is_enabled atomic callbacks only if the + * underlying SCMI transport for an SCMI instance is configured to + * handle SCMI commands in an atomic manner. + * + * When no SCMI atomic transport support is available we instead provide + * only the prepare/unprepare API, as allowed by the clock framework + * when atomic calls are not available. + */ + if (feats_key & BIT(SCMI_CLK_STATE_CTRL_SUPPORTED)) { + if (feats_key & BIT(SCMI_CLK_ATOMIC_SUPPORTED)) { + ops->enable = scmi_clk_atomic_enable; + ops->disable = scmi_clk_atomic_disable; + } else { + ops->prepare = scmi_clk_enable; + ops->unprepare = scmi_clk_disable; + } + } + + if (feats_key & BIT(SCMI_CLK_ATOMIC_SUPPORTED)) + ops->is_enabled = scmi_clk_atomic_is_enabled; + + /* Rate ops */ + ops->recalc_rate = scmi_clk_recalc_rate; + ops->round_rate = scmi_clk_round_rate; + ops->determine_rate = scmi_clk_determine_rate; + if (feats_key & BIT(SCMI_CLK_RATE_CTRL_SUPPORTED)) + ops->set_rate = scmi_clk_set_rate; + + /* Parent ops */ + ops->get_parent = scmi_clk_get_parent; + if (feats_key & BIT(SCMI_CLK_PARENT_CTRL_SUPPORTED)) + ops->set_parent = scmi_clk_set_parent; + + /* Duty cycle */ + if (feats_key & BIT(SCMI_CLK_DUTY_CYCLE_SUPPORTED)) { + ops->get_duty_cycle = scmi_clk_get_duty_cycle; + ops->set_duty_cycle = scmi_clk_set_duty_cycle; + } + + return ops; +} + +/** + * scmi_clk_ops_select() - Select a proper set of clock operations + * @sclk: A reference to an SCMI clock descriptor + * @atomic_capable: A flag to indicate if atomic mode is supported by the + * transport + * @atomic_threshold_us: Platform atomic threshold value in microseconds: + * clk_ops are atomic when clock enable latency is less + * than this threshold + * @clk_ops_db: A reference to the array used as a database to store all the + * created clock operations combinations. + * @db_size: Maximum number of entries held by @clk_ops_db + * + * After having built a bitmap descriptor to represent the set of features + * needed by this SCMI clock, at first use it to lookup into the set of + * previously allocated clk_ops to check if a suitable combination of clock + * operations was already created; when no match is found allocate a brand new + * set of clk_ops satisfying the required combination of features and save it + * for future references. + * + * In this way only one set of clk_ops is ever created for each different + * combination that is effectively needed by a driver instance. + * + * Return: A pointer to the allocated and configured clk_ops on success, or + * NULL otherwise. + */ +static const struct clk_ops * +scmi_clk_ops_select(struct scmi_clk *sclk, bool atomic_capable, + unsigned int atomic_threshold_us, + const struct clk_ops **clk_ops_db, size_t db_size) +{ + const struct scmi_clock_info *ci = sclk->info; + unsigned int feats_key = 0; + const struct clk_ops *ops; + + /* + * Note that when transport is atomic but SCMI protocol did not + * specify (or support) an enable_latency associated with a + * clock, we default to use atomic operations mode. + */ + if (atomic_capable && ci->enable_latency <= atomic_threshold_us) + feats_key |= BIT(SCMI_CLK_ATOMIC_SUPPORTED); + + if (!ci->state_ctrl_forbidden) + feats_key |= BIT(SCMI_CLK_STATE_CTRL_SUPPORTED); + + if (!ci->rate_ctrl_forbidden) + feats_key |= BIT(SCMI_CLK_RATE_CTRL_SUPPORTED); + + if (!ci->parent_ctrl_forbidden) + feats_key |= BIT(SCMI_CLK_PARENT_CTRL_SUPPORTED); + + if (ci->extended_config) + feats_key |= BIT(SCMI_CLK_DUTY_CYCLE_SUPPORTED); + + if (WARN_ON(feats_key >= db_size)) + return NULL; + + /* Lookup previously allocated ops */ + ops = clk_ops_db[feats_key]; + if (ops) + return ops; + + /* Did not find a pre-allocated clock_ops */ + ops = scmi_clk_ops_alloc(sclk->dev, feats_key); + if (!ops) + return NULL; + + /* Store new ops combinations */ + clk_ops_db[feats_key] = ops; + + return ops; +} + static int scmi_clocks_probe(struct scmi_device *sdev) { int idx, count, err; - unsigned int atomic_threshold; - bool is_atomic; + unsigned int atomic_threshold_us; + bool transport_is_atomic; struct clk_hw **hws; struct clk_hw_onecell_data *clk_data; struct device *dev = &sdev->dev; struct device_node *np = dev->of_node; const struct scmi_handle *handle = sdev->handle; struct scmi_protocol_handle *ph; + const struct clk_ops *scmi_clk_ops_db[SCMI_MAX_CLK_OPS] = {}; if (!handle) return -ENODEV; @@ -264,7 +415,8 @@ static int scmi_clocks_probe(struct scmi_device *sdev) clk_data->num = count; hws = clk_data->hws; - is_atomic = handle->is_transport_atomic(handle, &atomic_threshold); + transport_is_atomic = handle->is_transport_atomic(handle, + &atomic_threshold_us); for (idx = 0; idx < count; idx++) { struct scmi_clk *sclk; @@ -286,15 +438,17 @@ static int scmi_clocks_probe(struct scmi_device *sdev) sclk->dev = dev; /* - * Note that when transport is atomic but SCMI protocol did not - * specify (or support) an enable_latency associated with a - * clock, we default to use atomic operations mode. + * Note that the scmi_clk_ops_db is on the stack, not global, + * because it cannot be shared between mulitple probe-sequences + * to avoid sharing the devm_ allocated clk_ops between multiple + * SCMI clk driver instances. */ - if (is_atomic && - sclk->info->enable_latency <= atomic_threshold) - scmi_ops = &scmi_atomic_clk_ops; - else - scmi_ops = &scmi_clk_ops; + scmi_ops = scmi_clk_ops_select(sclk, transport_is_atomic, + atomic_threshold_us, + scmi_clk_ops_db, + ARRAY_SIZE(scmi_clk_ops_db)); + if (!scmi_ops) + return -ENOMEM; /* Initialize clock parent data. */ if (sclk->info->num_parents > 0) { @@ -318,8 +472,7 @@ static int scmi_clocks_probe(struct scmi_device *sdev) } else { dev_dbg(dev, "Registered clock:%s%s\n", sclk->info->name, - scmi_ops == &scmi_atomic_clk_ops ? - " (atomic ops)" : ""); + scmi_ops->enable ? " (atomic ops)" : ""); hws[idx] = &sclk->hw; } } diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c index 9cd80522ca2d..6a77d7e201a9 100644 --- a/drivers/clk/clkdev.c +++ b/drivers/clk/clkdev.c @@ -158,23 +158,54 @@ vclkdev_alloc(struct clk_hw *hw, const char *con_id, const char *dev_fmt, va_list ap) { struct clk_lookup_alloc *cla; + struct va_format vaf; + const char *failure; + va_list ap_copy; + size_t max_size; + ssize_t res; cla = kzalloc(sizeof(*cla), GFP_KERNEL); if (!cla) return NULL; + va_copy(ap_copy, ap); + cla->cl.clk_hw = hw; if (con_id) { - strscpy(cla->con_id, con_id, sizeof(cla->con_id)); + res = strscpy(cla->con_id, con_id, sizeof(cla->con_id)); + if (res < 0) { + max_size = sizeof(cla->con_id); + failure = "connection"; + goto fail; + } cla->cl.con_id = cla->con_id; } if (dev_fmt) { - vscnprintf(cla->dev_id, sizeof(cla->dev_id), dev_fmt, ap); + res = vsnprintf(cla->dev_id, sizeof(cla->dev_id), dev_fmt, ap); + if (res >= sizeof(cla->dev_id)) { + max_size = sizeof(cla->dev_id); + failure = "device"; + goto fail; + } cla->cl.dev_id = cla->dev_id; } + va_end(ap_copy); + return &cla->cl; + +fail: + if (dev_fmt) + vaf.fmt = dev_fmt; + else + vaf.fmt = "null-device"; + vaf.va = &ap_copy; + pr_err("%pV:%s: %s ID is greater than %zu\n", + &vaf, con_id, failure, max_size); + va_end(ap_copy); + kfree(cla); + return NULL; } static struct clk_lookup * diff --git a/drivers/clk/imx/Kconfig b/drivers/clk/imx/Kconfig index db3bca5f4ec9..6da0fba68225 100644 --- a/drivers/clk/imx/Kconfig +++ b/drivers/clk/imx/Kconfig @@ -114,6 +114,13 @@ config CLK_IMX93 help Build the driver for i.MX93 CCM Clock Driver +config CLK_IMX95_BLK_CTL + tristate "IMX95 Clock Driver for BLK CTL" + depends on ARCH_MXC || COMPILE_TEST + select MXC_CLK + help + Build the clock driver for i.MX95 BLK CTL + config CLK_IMXRT1050 tristate "IMXRT1050 CCM Clock Driver" depends on SOC_IMXRT || COMPILE_TEST diff --git a/drivers/clk/imx/Makefile b/drivers/clk/imx/Makefile index d4b8e10b1970..03f2b2a1ab63 100644 --- a/drivers/clk/imx/Makefile +++ b/drivers/clk/imx/Makefile @@ -31,6 +31,7 @@ obj-$(CONFIG_CLK_IMX8MP) += clk-imx8mp.o clk-imx8mp-audiomix.o obj-$(CONFIG_CLK_IMX8MQ) += clk-imx8mq.o obj-$(CONFIG_CLK_IMX93) += clk-imx93.o +obj-$(CONFIG_CLK_IMX95_BLK_CTL) += clk-imx95-blk-ctl.o obj-$(CONFIG_MXC_CLK_SCU) += clk-imx-scu.o clk-imx-lpcg-scu.o clk-imx-acm.o clk-imx-scu-$(CONFIG_CLK_IMX8QXP) += clk-scu.o clk-imx8qxp.o \ diff --git a/drivers/clk/imx/clk-imx8mp-audiomix.c b/drivers/clk/imx/clk-imx8mp-audiomix.c index 55ed211a5e0b..b381d6f784c8 100644 --- a/drivers/clk/imx/clk-imx8mp-audiomix.c +++ b/drivers/clk/imx/clk-imx8mp-audiomix.c @@ -7,10 +7,12 @@ #include <linux/clk-provider.h> #include <linux/device.h> +#include <linux/io.h> #include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include <dt-bindings/clock/imx8mp-clock.h> @@ -18,6 +20,7 @@ #define CLKEN0 0x000 #define CLKEN1 0x004 +#define EARC 0x200 #define SAI1_MCLK_SEL 0x300 #define SAI2_MCLK_SEL 0x304 #define SAI3_MCLK_SEL 0x308 @@ -26,6 +29,11 @@ #define SAI7_MCLK_SEL 0x314 #define PDM_SEL 0x318 #define SAI_PLL_GNRL_CTL 0x400 +#define SAI_PLL_FDIVL_CTL0 0x404 +#define SAI_PLL_FDIVL_CTL1 0x408 +#define SAI_PLL_SSCG_CTL 0x40C +#define SAI_PLL_MNIT_CTL 0x410 +#define IPG_LP_CTRL 0x504 #define SAIn_MCLK1_PARENT(n) \ static const struct clk_parent_data \ @@ -182,26 +190,82 @@ static struct clk_imx8mp_audiomix_sel sels[] = { CLK_SAIn(7) }; +static const u16 audiomix_regs[] = { + CLKEN0, + CLKEN1, + EARC, + SAI1_MCLK_SEL, + SAI2_MCLK_SEL, + SAI3_MCLK_SEL, + SAI5_MCLK_SEL, + SAI6_MCLK_SEL, + SAI7_MCLK_SEL, + PDM_SEL, + SAI_PLL_GNRL_CTL, + SAI_PLL_FDIVL_CTL0, + SAI_PLL_FDIVL_CTL1, + SAI_PLL_SSCG_CTL, + SAI_PLL_MNIT_CTL, + IPG_LP_CTRL, +}; + +struct clk_imx8mp_audiomix_priv { + void __iomem *base; + u32 regs_save[ARRAY_SIZE(audiomix_regs)]; + + /* Must be last */ + struct clk_hw_onecell_data clk_data; +}; + +static void clk_imx8mp_audiomix_save_restore(struct device *dev, bool save) +{ + struct clk_imx8mp_audiomix_priv *priv = dev_get_drvdata(dev); + void __iomem *base = priv->base; + int i; + + if (save) { + for (i = 0; i < ARRAY_SIZE(audiomix_regs); i++) + priv->regs_save[i] = readl(base + audiomix_regs[i]); + } else { + for (i = 0; i < ARRAY_SIZE(audiomix_regs); i++) + writel(priv->regs_save[i], base + audiomix_regs[i]); + } +} + static int clk_imx8mp_audiomix_probe(struct platform_device *pdev) { - struct clk_hw_onecell_data *priv; + struct clk_imx8mp_audiomix_priv *priv; + struct clk_hw_onecell_data *clk_hw_data; struct device *dev = &pdev->dev; void __iomem *base; struct clk_hw *hw; - int i; + int i, ret; priv = devm_kzalloc(dev, - struct_size(priv, hws, IMX8MP_CLK_AUDIOMIX_END), + struct_size(priv, clk_data.hws, IMX8MP_CLK_AUDIOMIX_END), GFP_KERNEL); if (!priv) return -ENOMEM; - priv->num = IMX8MP_CLK_AUDIOMIX_END; + clk_hw_data = &priv->clk_data; + clk_hw_data->num = IMX8MP_CLK_AUDIOMIX_END; base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); + priv->base = base; + dev_set_drvdata(dev, priv); + + /* + * pm_runtime_enable needs to be called before clk register. + * That is to make core->rpm_enabled to be true for clock + * usage. + */ + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + for (i = 0; i < ARRAY_SIZE(sels); i++) { if (sels[i].num_parents == 1) { hw = devm_clk_hw_register_gate_parent_data(dev, @@ -216,10 +280,12 @@ static int clk_imx8mp_audiomix_probe(struct platform_device *pdev) 0, NULL, NULL); } - if (IS_ERR(hw)) - return PTR_ERR(hw); + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto err_clk_register; + } - priv->hws[sels[i].clkid] = hw; + clk_hw_data->hws[sels[i].clkid] = hw; } /* SAI PLL */ @@ -228,39 +294,84 @@ static int clk_imx8mp_audiomix_probe(struct platform_device *pdev) ARRAY_SIZE(clk_imx8mp_audiomix_pll_parents), CLK_SET_RATE_NO_REPARENT, base + SAI_PLL_GNRL_CTL, 0, 2, 0, NULL, NULL); - priv->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_REF_SEL] = hw; + clk_hw_data->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_REF_SEL] = hw; hw = imx_dev_clk_hw_pll14xx(dev, "sai_pll", "sai_pll_ref_sel", base + 0x400, &imx_1443x_pll); - if (IS_ERR(hw)) - return PTR_ERR(hw); - priv->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL] = hw; + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto err_clk_register; + } + clk_hw_data->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL] = hw; hw = devm_clk_hw_register_mux_parent_data_table(dev, "sai_pll_bypass", clk_imx8mp_audiomix_pll_bypass_sels, ARRAY_SIZE(clk_imx8mp_audiomix_pll_bypass_sels), CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, base + SAI_PLL_GNRL_CTL, 16, 1, 0, NULL, NULL); - if (IS_ERR(hw)) - return PTR_ERR(hw); - priv->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_BYPASS] = hw; + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto err_clk_register; + } + + clk_hw_data->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_BYPASS] = hw; hw = devm_clk_hw_register_gate(dev, "sai_pll_out", "sai_pll_bypass", 0, base + SAI_PLL_GNRL_CTL, 13, 0, NULL); - if (IS_ERR(hw)) - return PTR_ERR(hw); - priv->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_OUT] = hw; + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto err_clk_register; + } + clk_hw_data->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_OUT] = hw; hw = devm_clk_hw_register_fixed_factor(dev, "sai_pll_out_div2", "sai_pll_out", 0, 1, 2); - if (IS_ERR(hw)) - return PTR_ERR(hw); + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto err_clk_register; + } + + ret = devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_onecell_get, + clk_hw_data); + if (ret) + goto err_clk_register; + + pm_runtime_put_sync(dev); + return 0; + +err_clk_register: + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + return ret; +} + +static void clk_imx8mp_audiomix_remove(struct platform_device *pdev) +{ + pm_runtime_disable(&pdev->dev); +} + +static int clk_imx8mp_audiomix_runtime_suspend(struct device *dev) +{ + clk_imx8mp_audiomix_save_restore(dev, true); - return devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_onecell_get, - priv); + return 0; } +static int clk_imx8mp_audiomix_runtime_resume(struct device *dev) +{ + clk_imx8mp_audiomix_save_restore(dev, false); + + return 0; +} + +static const struct dev_pm_ops clk_imx8mp_audiomix_pm_ops = { + RUNTIME_PM_OPS(clk_imx8mp_audiomix_runtime_suspend, + clk_imx8mp_audiomix_runtime_resume, NULL) + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) +}; + static const struct of_device_id clk_imx8mp_audiomix_of_match[] = { { .compatible = "fsl,imx8mp-audio-blk-ctrl" }, { /* sentinel */ } @@ -269,9 +380,11 @@ MODULE_DEVICE_TABLE(of, clk_imx8mp_audiomix_of_match); static struct platform_driver clk_imx8mp_audiomix_driver = { .probe = clk_imx8mp_audiomix_probe, + .remove_new = clk_imx8mp_audiomix_remove, .driver = { .name = "imx8mp-audio-blk-ctrl", .of_match_table = clk_imx8mp_audiomix_of_match, + .pm = pm_ptr(&clk_imx8mp_audiomix_pm_ops), }, }; diff --git a/drivers/clk/imx/clk-imx95-blk-ctl.c b/drivers/clk/imx/clk-imx95-blk-ctl.c new file mode 100644 index 000000000000..74f595f9e5e3 --- /dev/null +++ b/drivers/clk/imx/clk-imx95-blk-ctl.c @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2024 NXP + */ + +#include <dt-bindings/clock/nxp,imx95-clock.h> +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/pm_runtime.h> +#include <linux/debugfs.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/types.h> + +enum { + CLK_GATE, + CLK_DIVIDER, + CLK_MUX, +}; + +struct imx95_blk_ctl { + struct device *dev; + spinlock_t lock; + struct clk *clk_apb; + + void __iomem *base; + /* clock gate register */ + u32 clk_reg_restore; +}; + +struct imx95_blk_ctl_clk_dev_data { + const char *name; + const char * const *parent_names; + u32 num_parents; + u32 reg; + u32 bit_idx; + u32 bit_width; + u32 clk_type; + u32 flags; + u32 flags2; + u32 type; +}; + +struct imx95_blk_ctl_dev_data { + const struct imx95_blk_ctl_clk_dev_data *clk_dev_data; + u32 num_clks; + bool rpm_enabled; + u32 clk_reg_offset; +}; + +static const struct imx95_blk_ctl_clk_dev_data vpublk_clk_dev_data[] = { + [IMX95_CLK_VPUBLK_WAVE] = { + .name = "vpublk_wave_vpu", + .parent_names = (const char *[]){ "vpu", }, + .num_parents = 1, + .reg = 8, + .bit_idx = 0, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_VPUBLK_JPEG_ENC] = { + .name = "vpublk_jpeg_enc", + .parent_names = (const char *[]){ "vpujpeg", }, + .num_parents = 1, + .reg = 8, + .bit_idx = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_VPUBLK_JPEG_DEC] = { + .name = "vpublk_jpeg_dec", + .parent_names = (const char *[]){ "vpujpeg", }, + .num_parents = 1, + .reg = 8, + .bit_idx = 2, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + } +}; + +static const struct imx95_blk_ctl_dev_data vpublk_dev_data = { + .num_clks = ARRAY_SIZE(vpublk_clk_dev_data), + .clk_dev_data = vpublk_clk_dev_data, + .rpm_enabled = true, + .clk_reg_offset = 8, +}; + +static const struct imx95_blk_ctl_clk_dev_data camblk_clk_dev_data[] = { + [IMX95_CLK_CAMBLK_CSI2_FOR0] = { + .name = "camblk_csi2_for0", + .parent_names = (const char *[]){ "camisi", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 0, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_CAMBLK_CSI2_FOR1] = { + .name = "camblk_csi2_for1", + .parent_names = (const char *[]){ "camisi", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_CAMBLK_ISP_AXI] = { + .name = "camblk_isp_axi", + .parent_names = (const char *[]){ "camaxi", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 4, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_CAMBLK_ISP_PIXEL] = { + .name = "camblk_isp_pixel", + .parent_names = (const char *[]){ "camisi", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 5, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_CAMBLK_ISP] = { + .name = "camblk_isp", + .parent_names = (const char *[]){ "camisi", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 6, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + } +}; + +static const struct imx95_blk_ctl_dev_data camblk_dev_data = { + .num_clks = ARRAY_SIZE(camblk_clk_dev_data), + .clk_dev_data = camblk_clk_dev_data, + .clk_reg_offset = 0, +}; + +static const struct imx95_blk_ctl_clk_dev_data lvds_clk_dev_data[] = { + [IMX95_CLK_DISPMIX_LVDS_PHY_DIV] = { + .name = "ldb_phy_div", + .parent_names = (const char *[]){ "ldbpll", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 0, + .bit_width = 1, + .type = CLK_DIVIDER, + .flags2 = CLK_DIVIDER_POWER_OF_TWO, + }, + [IMX95_CLK_DISPMIX_LVDS_CH0_GATE] = { + .name = "lvds_ch0_gate", + .parent_names = (const char *[]){ "ldb_phy_div", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 1, + .bit_width = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_DISPMIX_LVDS_CH1_GATE] = { + .name = "lvds_ch1_gate", + .parent_names = (const char *[]){ "ldb_phy_div", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 2, + .bit_width = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_DISPMIX_PIX_DI0_GATE] = { + .name = "lvds_di0_gate", + .parent_names = (const char *[]){ "ldb_pll_div7", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 3, + .bit_width = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_DISPMIX_PIX_DI1_GATE] = { + .name = "lvds_di1_gate", + .parent_names = (const char *[]){ "ldb_pll_div7", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 4, + .bit_width = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, +}; + +static const struct imx95_blk_ctl_dev_data lvds_csr_dev_data = { + .num_clks = ARRAY_SIZE(lvds_clk_dev_data), + .clk_dev_data = lvds_clk_dev_data, + .clk_reg_offset = 0, +}; + +static const struct imx95_blk_ctl_clk_dev_data dispmix_csr_clk_dev_data[] = { + [IMX95_CLK_DISPMIX_ENG0_SEL] = { + .name = "disp_engine0_sel", + .parent_names = (const char *[]){"videopll1", "dsi_pll", "ldb_pll_div7", }, + .num_parents = 4, + .reg = 0, + .bit_idx = 0, + .bit_width = 2, + .type = CLK_MUX, + .flags = CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, + }, + [IMX95_CLK_DISPMIX_ENG1_SEL] = { + .name = "disp_engine1_sel", + .parent_names = (const char *[]){"videopll1", "dsi_pll", "ldb_pll_div7", }, + .num_parents = 4, + .reg = 0, + .bit_idx = 2, + .bit_width = 2, + .type = CLK_MUX, + .flags = CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, + } +}; + +static const struct imx95_blk_ctl_dev_data dispmix_csr_dev_data = { + .num_clks = ARRAY_SIZE(dispmix_csr_clk_dev_data), + .clk_dev_data = dispmix_csr_clk_dev_data, + .clk_reg_offset = 0, +}; + +static int imx95_bc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct imx95_blk_ctl_dev_data *bc_data; + struct imx95_blk_ctl *bc; + struct clk_hw_onecell_data *clk_hw_data; + struct clk_hw **hws; + void __iomem *base; + int i, ret; + + bc = devm_kzalloc(dev, sizeof(*bc), GFP_KERNEL); + if (!bc) + return -ENOMEM; + bc->dev = dev; + dev_set_drvdata(&pdev->dev, bc); + + spin_lock_init(&bc->lock); + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + bc->base = base; + bc->clk_apb = devm_clk_get(dev, NULL); + if (IS_ERR(bc->clk_apb)) + return dev_err_probe(dev, PTR_ERR(bc->clk_apb), "failed to get APB clock\n"); + + ret = clk_prepare_enable(bc->clk_apb); + if (ret) { + dev_err(dev, "failed to enable apb clock: %d\n", ret); + return ret; + } + + bc_data = of_device_get_match_data(dev); + if (!bc_data) + return devm_of_platform_populate(dev); + + clk_hw_data = devm_kzalloc(dev, struct_size(clk_hw_data, hws, bc_data->num_clks), + GFP_KERNEL); + if (!clk_hw_data) + return -ENOMEM; + + if (bc_data->rpm_enabled) + pm_runtime_enable(&pdev->dev); + + clk_hw_data->num = bc_data->num_clks; + hws = clk_hw_data->hws; + + for (i = 0; i < bc_data->num_clks; i++) { + const struct imx95_blk_ctl_clk_dev_data *data = &bc_data->clk_dev_data[i]; + void __iomem *reg = base + data->reg; + + if (data->type == CLK_MUX) { + hws[i] = clk_hw_register_mux(dev, data->name, data->parent_names, + data->num_parents, data->flags, reg, + data->bit_idx, data->bit_width, + data->flags2, &bc->lock); + } else if (data->type == CLK_DIVIDER) { + hws[i] = clk_hw_register_divider(dev, data->name, data->parent_names[0], + data->flags, reg, data->bit_idx, + data->bit_width, data->flags2, &bc->lock); + } else { + hws[i] = clk_hw_register_gate(dev, data->name, data->parent_names[0], + data->flags, reg, data->bit_idx, + data->flags2, &bc->lock); + } + if (IS_ERR(hws[i])) { + ret = PTR_ERR(hws[i]); + dev_err(dev, "failed to register: %s:%d\n", data->name, ret); + goto cleanup; + } + } + + ret = of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get, clk_hw_data); + if (ret) + goto cleanup; + + ret = devm_of_platform_populate(dev); + if (ret) { + of_clk_del_provider(dev->of_node); + goto cleanup; + } + + if (pm_runtime_enabled(bc->dev)) + clk_disable_unprepare(bc->clk_apb); + + return 0; + +cleanup: + for (i = 0; i < bc_data->num_clks; i++) { + if (IS_ERR_OR_NULL(hws[i])) + continue; + clk_hw_unregister(hws[i]); + } + + if (bc_data->rpm_enabled) + pm_runtime_disable(&pdev->dev); + + return ret; +} + +#ifdef CONFIG_PM +static int imx95_bc_runtime_suspend(struct device *dev) +{ + struct imx95_blk_ctl *bc = dev_get_drvdata(dev); + + clk_disable_unprepare(bc->clk_apb); + return 0; +} + +static int imx95_bc_runtime_resume(struct device *dev) +{ + struct imx95_blk_ctl *bc = dev_get_drvdata(dev); + + return clk_prepare_enable(bc->clk_apb); +} +#endif + +#ifdef CONFIG_PM_SLEEP +static int imx95_bc_suspend(struct device *dev) +{ + struct imx95_blk_ctl *bc = dev_get_drvdata(dev); + const struct imx95_blk_ctl_dev_data *bc_data; + int ret; + + bc_data = of_device_get_match_data(dev); + if (!bc_data) + return 0; + + if (bc_data->rpm_enabled) { + ret = pm_runtime_get_sync(bc->dev); + if (ret < 0) { + pm_runtime_put_noidle(bc->dev); + return ret; + } + } + + bc->clk_reg_restore = readl(bc->base + bc_data->clk_reg_offset); + + return 0; +} + +static int imx95_bc_resume(struct device *dev) +{ + struct imx95_blk_ctl *bc = dev_get_drvdata(dev); + const struct imx95_blk_ctl_dev_data *bc_data; + + bc_data = of_device_get_match_data(dev); + if (!bc_data) + return 0; + + writel(bc->clk_reg_restore, bc->base + bc_data->clk_reg_offset); + + if (bc_data->rpm_enabled) + pm_runtime_put(bc->dev); + + return 0; +} +#endif + +static const struct dev_pm_ops imx95_bc_pm_ops = { + SET_RUNTIME_PM_OPS(imx95_bc_runtime_suspend, imx95_bc_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(imx95_bc_suspend, imx95_bc_resume) +}; + +static const struct of_device_id imx95_bc_of_match[] = { + { .compatible = "nxp,imx95-camera-csr", .data = &camblk_dev_data }, + { .compatible = "nxp,imx95-display-master-csr", }, + { .compatible = "nxp,imx95-lvds-csr", .data = &lvds_csr_dev_data }, + { .compatible = "nxp,imx95-display-csr", .data = &dispmix_csr_dev_data }, + { .compatible = "nxp,imx95-vpu-csr", .data = &vpublk_dev_data }, + { /* Sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, imx95_bc_of_match); + +static struct platform_driver imx95_bc_driver = { + .probe = imx95_bc_probe, + .driver = { + .name = "imx95-blk-ctl", + .of_match_table = imx95_bc_of_match, + .pm = &imx95_bc_pm_ops, + }, +}; +module_platform_driver(imx95_bc_driver); + +MODULE_DESCRIPTION("NXP i.MX95 blk ctl driver"); +MODULE_AUTHOR("Peng Fan <peng.fan@nxp.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8365-mm.c b/drivers/clk/mediatek/clk-mt8365-mm.c index 01a2ef8f594e..3f62ec750733 100644 --- a/drivers/clk/mediatek/clk-mt8365-mm.c +++ b/drivers/clk/mediatek/clk-mt8365-mm.c @@ -53,7 +53,7 @@ static const struct mtk_gate mm_clks[] = { GATE_MM0(CLK_MM_MM_DSI0, "mm_dsi0", "mm_sel", 17), GATE_MM0(CLK_MM_MM_DISP_RDMA1, "mm_disp_rdma1", "mm_sel", 18), GATE_MM0(CLK_MM_MM_MDP_RDMA1, "mm_mdp_rdma1", "mm_sel", 19), - GATE_MM0(CLK_MM_DPI0_DPI0, "mm_dpi0_dpi0", "vpll_dpix", 20), + GATE_MM0(CLK_MM_DPI0_DPI0, "mm_dpi0_dpi0", "dpi0_sel", 20), GATE_MM0(CLK_MM_MM_FAKE, "mm_fake", "mm_sel", 21), GATE_MM0(CLK_MM_MM_SMI_COMMON, "mm_smi_common", "mm_sel", 22), GATE_MM0(CLK_MM_MM_SMI_LARB0, "mm_smi_larb0", "mm_sel", 23), diff --git a/drivers/clk/mediatek/clk-pllfh.c b/drivers/clk/mediatek/clk-pllfh.c index 3a2b3f90be25..094ec8a26d66 100644 --- a/drivers/clk/mediatek/clk-pllfh.c +++ b/drivers/clk/mediatek/clk-pllfh.c @@ -68,7 +68,7 @@ void fhctl_parse_dt(const u8 *compatible_node, struct mtk_pllfh_data *pllfhs, node = of_find_compatible_node(NULL, NULL, compatible_node); if (!node) { - pr_err("cannot find \"%s\"\n", compatible_node); + pr_warn("cannot find \"%s\"\n", compatible_node); return; } diff --git a/drivers/clk/meson/Kconfig b/drivers/clk/meson/Kconfig index 29ffd14d267b..59a40a49f8e1 100644 --- a/drivers/clk/meson/Kconfig +++ b/drivers/clk/meson/Kconfig @@ -30,6 +30,10 @@ config COMMON_CLK_MESON_VID_PLL_DIV tristate select COMMON_CLK_MESON_REGMAP +config COMMON_CLK_MESON_VCLK + tristate + select COMMON_CLK_MESON_REGMAP + config COMMON_CLK_MESON_CLKC_UTILS tristate @@ -140,6 +144,7 @@ config COMMON_CLK_G12A select COMMON_CLK_MESON_EE_CLKC select COMMON_CLK_MESON_CPU_DYNDIV select COMMON_CLK_MESON_VID_PLL_DIV + select COMMON_CLK_MESON_VCLK select MFD_SYSCON help Support for the clock controller on Amlogic S905D2, S905X2 and S905Y2 diff --git a/drivers/clk/meson/Makefile b/drivers/clk/meson/Makefile index 9ee4b954c896..9ba43fe7a07a 100644 --- a/drivers/clk/meson/Makefile +++ b/drivers/clk/meson/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_COMMON_CLK_MESON_PLL) += clk-pll.o obj-$(CONFIG_COMMON_CLK_MESON_REGMAP) += clk-regmap.o obj-$(CONFIG_COMMON_CLK_MESON_SCLK_DIV) += sclk-div.o obj-$(CONFIG_COMMON_CLK_MESON_VID_PLL_DIV) += vid-pll-div.o +obj-$(CONFIG_COMMON_CLK_MESON_VCLK) += vclk.o # Amlogic Clock controllers diff --git a/drivers/clk/meson/a1-peripherals.c b/drivers/clk/meson/a1-peripherals.c index e2a1f12f9175..621af1e6e4b2 100644 --- a/drivers/clk/meson/a1-peripherals.c +++ b/drivers/clk/meson/a1-peripherals.c @@ -2187,6 +2187,7 @@ static struct regmap_config a1_periphs_regmap_cfg = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .max_register = DMC_CLK_CTRL, }; static struct meson_clk_hw_data a1_periphs_clks = { diff --git a/drivers/clk/meson/a1-pll.c b/drivers/clk/meson/a1-pll.c index 4325e8a6a3ef..90b0aeeb049c 100644 --- a/drivers/clk/meson/a1-pll.c +++ b/drivers/clk/meson/a1-pll.c @@ -299,6 +299,7 @@ static struct regmap_config a1_pll_regmap_cfg = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .max_register = ANACTRL_HIFIPLL_STS, }; static struct meson_clk_hw_data a1_pll_clks = { diff --git a/drivers/clk/meson/axg-aoclk.c b/drivers/clk/meson/axg-aoclk.c index d80ab4728f7a..e4d0f46f47f5 100644 --- a/drivers/clk/meson/axg-aoclk.c +++ b/drivers/clk/meson/axg-aoclk.c @@ -340,4 +340,4 @@ static struct platform_driver axg_aoclkc_driver = { }; module_platform_driver(axg_aoclkc_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/axg-audio.c b/drivers/clk/meson/axg-audio.c index ac3482960903..e03a5bf899c0 100644 --- a/drivers/clk/meson/axg-audio.c +++ b/drivers/clk/meson/axg-audio.c @@ -1877,4 +1877,4 @@ module_platform_driver(axg_audio_driver); MODULE_DESCRIPTION("Amlogic AXG/G12A/SM1 Audio Clock driver"); MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/axg.c b/drivers/clk/meson/axg.c index 5f60f2bcca59..52d610110e44 100644 --- a/drivers/clk/meson/axg.c +++ b/drivers/clk/meson/axg.c @@ -2185,4 +2185,4 @@ static struct platform_driver axg_driver = { }; module_platform_driver(axg_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-cpu-dyndiv.c b/drivers/clk/meson/clk-cpu-dyndiv.c index 8778c149d26a..aa824b030cb8 100644 --- a/drivers/clk/meson/clk-cpu-dyndiv.c +++ b/drivers/clk/meson/clk-cpu-dyndiv.c @@ -69,4 +69,4 @@ EXPORT_SYMBOL_GPL(meson_clk_cpu_dyndiv_ops); MODULE_DESCRIPTION("Amlogic CPU Dynamic Clock divider"); MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-dualdiv.c b/drivers/clk/meson/clk-dualdiv.c index feae49a8f6dc..d46c02b51be5 100644 --- a/drivers/clk/meson/clk-dualdiv.c +++ b/drivers/clk/meson/clk-dualdiv.c @@ -140,4 +140,4 @@ EXPORT_SYMBOL_GPL(meson_clk_dualdiv_ro_ops); MODULE_DESCRIPTION("Amlogic dual divider driver"); MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>"); MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-mpll.c b/drivers/clk/meson/clk-mpll.c index 20255e129b37..eae9b7dc5a6c 100644 --- a/drivers/clk/meson/clk-mpll.c +++ b/drivers/clk/meson/clk-mpll.c @@ -177,4 +177,4 @@ EXPORT_SYMBOL_GPL(meson_clk_mpll_ops); MODULE_DESCRIPTION("Amlogic MPLL driver"); MODULE_AUTHOR("Michael Turquette <mturquette@baylibre.com>"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-phase.c b/drivers/clk/meson/clk-phase.c index a6763439f7d2..ff3f0b1a3ed1 100644 --- a/drivers/clk/meson/clk-phase.c +++ b/drivers/clk/meson/clk-phase.c @@ -183,4 +183,4 @@ EXPORT_SYMBOL_GPL(meson_sclk_ws_inv_ops); MODULE_DESCRIPTION("Amlogic phase driver"); MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-pll.c b/drivers/clk/meson/clk-pll.c index 6fa7639a3050..07db8b5c3000 100644 --- a/drivers/clk/meson/clk-pll.c +++ b/drivers/clk/meson/clk-pll.c @@ -436,8 +436,8 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, ret = meson_clk_pll_enable(hw); if (ret) { - pr_warn("%s: pll did not lock, trying to restore old rate %lu\n", - __func__, old_rate); + pr_warn("%s: pll %s didn't lock, trying to set old rate %lu\n", + __func__, clk_hw_get_name(hw), old_rate); /* * FIXME: Do we really need/want this HACK ? * It looks unsafe. what happens if the clock gets into a @@ -486,4 +486,4 @@ EXPORT_SYMBOL_GPL(meson_clk_pll_ro_ops); MODULE_DESCRIPTION("Amlogic PLL driver"); MODULE_AUTHOR("Carlo Caione <carlo@endlessm.com>"); MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-regmap.c b/drivers/clk/meson/clk-regmap.c index 8ad8977cf1c2..ad116d24f700 100644 --- a/drivers/clk/meson/clk-regmap.c +++ b/drivers/clk/meson/clk-regmap.c @@ -183,4 +183,4 @@ EXPORT_SYMBOL_GPL(clk_regmap_mux_ro_ops); MODULE_DESCRIPTION("Amlogic regmap backed clock driver"); MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/g12a-aoclk.c b/drivers/clk/meson/g12a-aoclk.c index c6b1d55cd7c8..58976ed8b92a 100644 --- a/drivers/clk/meson/g12a-aoclk.c +++ b/drivers/clk/meson/g12a-aoclk.c @@ -475,4 +475,4 @@ static struct platform_driver g12a_aoclkc_driver = { }; module_platform_driver(g12a_aoclkc_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index 90f4c6103014..56e66ecc306e 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -22,6 +22,7 @@ #include "clk-regmap.h" #include "clk-cpu-dyndiv.h" #include "vid-pll-div.h" +#include "vclk.h" #include "meson-eeclk.h" #include "g12a.h" @@ -3165,7 +3166,7 @@ static struct clk_regmap g12a_vclk2_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = g12a_vclk_parent_hws, .num_parents = ARRAY_SIZE(g12a_vclk_parent_hws), - .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + .flags = CLK_SET_RATE_NO_REPARENT, }, }; @@ -3193,7 +3194,6 @@ static struct clk_regmap g12a_vclk2_input = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2_sel.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, }, }; @@ -3215,19 +3215,32 @@ static struct clk_regmap g12a_vclk_div = { }; static struct clk_regmap g12a_vclk2_div = { - .data = &(struct clk_regmap_div_data){ - .offset = HHI_VIID_CLK_DIV, - .shift = 0, - .width = 8, + .data = &(struct meson_vclk_div_data){ + .div = { + .reg_off = HHI_VIID_CLK_DIV, + .shift = 0, + .width = 8, + }, + .enable = { + .reg_off = HHI_VIID_CLK_DIV, + .shift = 16, + .width = 1, + }, + .reset = { + .reg_off = HHI_VIID_CLK_DIV, + .shift = 17, + .width = 1, + }, + .flags = CLK_DIVIDER_ROUND_CLOSEST, }, .hw.init = &(struct clk_init_data){ .name = "vclk2_div", - .ops = &clk_regmap_divider_ops, + .ops = &meson_vclk_div_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2_input.hw }, .num_parents = 1, - .flags = CLK_GET_RATE_NOCACHE, + .flags = CLK_SET_RATE_GATE, }, }; @@ -3246,16 +3259,24 @@ static struct clk_regmap g12a_vclk = { }; static struct clk_regmap g12a_vclk2 = { - .data = &(struct clk_regmap_gate_data){ - .offset = HHI_VIID_CLK_CNTL, - .bit_idx = 19, + .data = &(struct meson_vclk_gate_data){ + .enable = { + .reg_off = HHI_VIID_CLK_CNTL, + .shift = 19, + .width = 1, + }, + .reset = { + .reg_off = HHI_VIID_CLK_CNTL, + .shift = 15, + .width = 1, + }, }, .hw.init = &(struct clk_init_data) { .name = "vclk2", - .ops = &clk_regmap_gate_ops, + .ops = &meson_vclk_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2_div.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3339,7 +3360,7 @@ static struct clk_regmap g12a_vclk2_div1 = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3353,7 +3374,7 @@ static struct clk_regmap g12a_vclk2_div2_en = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3367,7 +3388,7 @@ static struct clk_regmap g12a_vclk2_div4_en = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3381,7 +3402,7 @@ static struct clk_regmap g12a_vclk2_div6_en = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3395,7 +3416,7 @@ static struct clk_regmap g12a_vclk2_div12_en = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3461,6 +3482,7 @@ static struct clk_fixed_factor g12a_vclk2_div2 = { &g12a_vclk2_div2_en.hw }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3474,6 +3496,7 @@ static struct clk_fixed_factor g12a_vclk2_div4 = { &g12a_vclk2_div4_en.hw }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3487,6 +3510,7 @@ static struct clk_fixed_factor g12a_vclk2_div6 = { &g12a_vclk2_div6_en.hw }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3500,6 +3524,7 @@ static struct clk_fixed_factor g12a_vclk2_div12 = { &g12a_vclk2_div12_en.hw }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3561,7 +3586,7 @@ static struct clk_regmap g12a_cts_encl_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = g12a_cts_parent_hws, .num_parents = ARRAY_SIZE(g12a_cts_parent_hws), - .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + .flags = CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, }, }; @@ -3717,15 +3742,26 @@ static struct clk_regmap g12a_mipi_dsi_pxclk_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = g12a_mipi_dsi_pxclk_parent_hws, .num_parents = ARRAY_SIZE(g12a_mipi_dsi_pxclk_parent_hws), - .flags = CLK_SET_RATE_NO_REPARENT, + .flags = CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, }, }; +/* + * FIXME: Force as bypass by forcing a single /1 table entry, and doensn't on boot value + * when setting a clock whith this node in the clock path, but doesn't garantee the divider + * is at /1 at boot until a rate is set. + */ +static const struct clk_div_table g12a_mipi_dsi_pxclk_div_table[] = { + { .val = 0, .div = 1 }, + { /* sentinel */ }, +}; + static struct clk_regmap g12a_mipi_dsi_pxclk_div = { .data = &(struct clk_regmap_div_data){ .offset = HHI_MIPIDSI_PHY_CLK_CNTL, .shift = 0, .width = 7, + .table = g12a_mipi_dsi_pxclk_div_table, }, .hw.init = &(struct clk_init_data){ .name = "mipi_dsi_pxclk_div", @@ -5578,4 +5614,4 @@ static struct platform_driver g12a_driver = { }; module_platform_driver(g12a_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/gxbb-aoclk.c b/drivers/clk/meson/gxbb-aoclk.c index 4aec1740ac34..dbda563729db 100644 --- a/drivers/clk/meson/gxbb-aoclk.c +++ b/drivers/clk/meson/gxbb-aoclk.c @@ -300,4 +300,4 @@ static struct platform_driver gxbb_aoclkc_driver = { }, }; module_platform_driver(gxbb_aoclkc_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index 1b1279d94781..29507b8c4304 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -3569,4 +3569,4 @@ static struct platform_driver gxbb_driver = { }; module_platform_driver(gxbb_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/meson-aoclk.c b/drivers/clk/meson/meson-aoclk.c index bf466fef263c..b8a9d59e6726 100644 --- a/drivers/clk/meson/meson-aoclk.c +++ b/drivers/clk/meson/meson-aoclk.c @@ -89,4 +89,4 @@ int meson_aoclkc_probe(struct platform_device *pdev) return devm_of_clk_add_hw_provider(dev, meson_clk_hw_get, (void *)&data->hw_clks); } EXPORT_SYMBOL_GPL(meson_aoclkc_probe); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/meson-eeclk.c b/drivers/clk/meson/meson-eeclk.c index 845ca8bfa346..3cbc7f233bba 100644 --- a/drivers/clk/meson/meson-eeclk.c +++ b/drivers/clk/meson/meson-eeclk.c @@ -58,4 +58,4 @@ int meson_eeclkc_probe(struct platform_device *pdev) return devm_of_clk_add_hw_provider(dev, meson_clk_hw_get, (void *)&data->hw_clks); } EXPORT_SYMBOL_GPL(meson_eeclkc_probe); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/s4-peripherals.c b/drivers/clk/meson/s4-peripherals.c index 6c35de3d536f..5e17ca50ab09 100644 --- a/drivers/clk/meson/s4-peripherals.c +++ b/drivers/clk/meson/s4-peripherals.c @@ -3751,6 +3751,7 @@ static struct regmap_config clkc_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .max_register = CLKCTRL_DEMOD_CLK_CTRL, }; static struct meson_clk_hw_data s4_periphs_clks = { @@ -3799,6 +3800,7 @@ static const struct of_device_id clkc_match_table[] = { }, {} }; +MODULE_DEVICE_TABLE(of, clkc_match_table); static struct platform_driver s4_driver = { .probe = meson_s4_periphs_probe, diff --git a/drivers/clk/meson/s4-pll.c b/drivers/clk/meson/s4-pll.c index 8dfaeccaadc2..d2650d96400c 100644 --- a/drivers/clk/meson/s4-pll.c +++ b/drivers/clk/meson/s4-pll.c @@ -798,6 +798,7 @@ static struct regmap_config clkc_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .max_register = ANACTRL_HDMIPLL_CTRL0, }; static struct meson_clk_hw_data s4_pll_clks = { @@ -853,6 +854,7 @@ static const struct of_device_id clkc_match_table[] = { }, {} }; +MODULE_DEVICE_TABLE(of, clkc_match_table); static struct platform_driver s4_driver = { .probe = meson_s4_pll_probe, diff --git a/drivers/clk/meson/sclk-div.c b/drivers/clk/meson/sclk-div.c index d12c45c4c261..987f5b06587c 100644 --- a/drivers/clk/meson/sclk-div.c +++ b/drivers/clk/meson/sclk-div.c @@ -251,4 +251,4 @@ EXPORT_SYMBOL_GPL(meson_sclk_div_ops); MODULE_DESCRIPTION("Amlogic Sample divider driver"); MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/vclk.c b/drivers/clk/meson/vclk.c new file mode 100644 index 000000000000..e886df55d6e3 --- /dev/null +++ b/drivers/clk/meson/vclk.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024 Neil Armstrong <neil.armstrong@linaro.org> + */ + +#include <linux/module.h> +#include "vclk.h" + +/* The VCLK gate has a supplementary reset bit to pulse after ungating */ + +static inline struct meson_vclk_gate_data * +clk_get_meson_vclk_gate_data(struct clk_regmap *clk) +{ + return (struct meson_vclk_gate_data *)clk->data; +} + +static int meson_vclk_gate_enable(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_gate_data *vclk = clk_get_meson_vclk_gate_data(clk); + + meson_parm_write(clk->map, &vclk->enable, 1); + + /* Do a reset pulse */ + meson_parm_write(clk->map, &vclk->reset, 1); + meson_parm_write(clk->map, &vclk->reset, 0); + + return 0; +} + +static void meson_vclk_gate_disable(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_gate_data *vclk = clk_get_meson_vclk_gate_data(clk); + + meson_parm_write(clk->map, &vclk->enable, 0); +} + +static int meson_vclk_gate_is_enabled(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_gate_data *vclk = clk_get_meson_vclk_gate_data(clk); + + return meson_parm_read(clk->map, &vclk->enable); +} + +const struct clk_ops meson_vclk_gate_ops = { + .enable = meson_vclk_gate_enable, + .disable = meson_vclk_gate_disable, + .is_enabled = meson_vclk_gate_is_enabled, +}; +EXPORT_SYMBOL_GPL(meson_vclk_gate_ops); + +/* The VCLK Divider has supplementary reset & enable bits */ + +static inline struct meson_vclk_div_data * +clk_get_meson_vclk_div_data(struct clk_regmap *clk) +{ + return (struct meson_vclk_div_data *)clk->data; +} + +static unsigned long meson_vclk_div_recalc_rate(struct clk_hw *hw, + unsigned long prate) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + + return divider_recalc_rate(hw, prate, meson_parm_read(clk->map, &vclk->div), + vclk->table, vclk->flags, vclk->div.width); +} + +static int meson_vclk_div_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + + return divider_determine_rate(hw, req, vclk->table, vclk->div.width, + vclk->flags); +} + +static int meson_vclk_div_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + int ret; + + ret = divider_get_val(rate, parent_rate, vclk->table, vclk->div.width, + vclk->flags); + if (ret < 0) + return ret; + + meson_parm_write(clk->map, &vclk->div, ret); + + return 0; +}; + +static int meson_vclk_div_enable(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + + /* Unreset the divider when ungating */ + meson_parm_write(clk->map, &vclk->reset, 0); + meson_parm_write(clk->map, &vclk->enable, 1); + + return 0; +} + +static void meson_vclk_div_disable(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + + /* Reset the divider when gating */ + meson_parm_write(clk->map, &vclk->enable, 0); + meson_parm_write(clk->map, &vclk->reset, 1); +} + +static int meson_vclk_div_is_enabled(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + + return meson_parm_read(clk->map, &vclk->enable); +} + +const struct clk_ops meson_vclk_div_ops = { + .recalc_rate = meson_vclk_div_recalc_rate, + .determine_rate = meson_vclk_div_determine_rate, + .set_rate = meson_vclk_div_set_rate, + .enable = meson_vclk_div_enable, + .disable = meson_vclk_div_disable, + .is_enabled = meson_vclk_div_is_enabled, +}; +EXPORT_SYMBOL_GPL(meson_vclk_div_ops); + +MODULE_DESCRIPTION("Amlogic vclk clock driver"); +MODULE_AUTHOR("Neil Armstrong <neil.armstrong@linaro.org>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/vclk.h b/drivers/clk/meson/vclk.h new file mode 100644 index 000000000000..20b0b181db09 --- /dev/null +++ b/drivers/clk/meson/vclk.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2024 Neil Armstrong <neil.armstrong@linaro.org> + */ + +#ifndef __VCLK_H +#define __VCLK_H + +#include "clk-regmap.h" +#include "parm.h" + +/** + * struct meson_vclk_gate_data - vclk_gate regmap backed specific data + * + * @enable: vclk enable field + * @reset: vclk reset field + * @flags: hardware-specific flags + * + * Flags: + * Same as clk_gate except CLK_GATE_HIWORD_MASK which is ignored + */ +struct meson_vclk_gate_data { + struct parm enable; + struct parm reset; + u8 flags; +}; + +extern const struct clk_ops meson_vclk_gate_ops; + +/** + * struct meson_vclk_div_data - vclk_div regmap back specific data + * + * @div: divider field + * @enable: vclk divider enable field + * @reset: vclk divider reset field + * @table: array of value/divider pairs, last entry should have div = 0 + * + * Flags: + * Same as clk_divider except CLK_DIVIDER_HIWORD_MASK which is ignored + */ +struct meson_vclk_div_data { + struct parm div; + struct parm enable; + struct parm reset; + const struct clk_div_table *table; + u8 flags; +}; + +extern const struct clk_ops meson_vclk_div_ops; + +#endif /* __VCLK_H */ diff --git a/drivers/clk/meson/vid-pll-div.c b/drivers/clk/meson/vid-pll-div.c index daff235bc763..ee129f86794d 100644 --- a/drivers/clk/meson/vid-pll-div.c +++ b/drivers/clk/meson/vid-pll-div.c @@ -96,4 +96,4 @@ EXPORT_SYMBOL_GPL(meson_vid_pll_div_ro_ops); MODULE_DESCRIPTION("Amlogic video pll divider driver"); MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/microchip/clk-mpfs.c b/drivers/clk/microchip/clk-mpfs.c index 22eab91a6712..28ec0da88cb3 100644 --- a/drivers/clk/microchip/clk-mpfs.c +++ b/drivers/clk/microchip/clk-mpfs.c @@ -4,12 +4,10 @@ * * Copyright (C) 2020-2022 Microchip Technology Inc. All rights reserved. */ -#include <linux/auxiliary_bus.h> #include <linux/clk-provider.h> #include <linux/io.h> #include <linux/module.h> #include <linux/platform_device.h> -#include <linux/slab.h> #include <dt-bindings/clock/microchip,mpfs-clock.h> #include <soc/microchip/mpfs.h> @@ -361,93 +359,6 @@ static int mpfs_clk_register_periphs(struct device *dev, struct mpfs_periph_hw_c return 0; } -/* - * Peripheral clock resets - */ - -#if IS_ENABLED(CONFIG_RESET_CONTROLLER) - -u32 mpfs_reset_read(struct device *dev) -{ - struct mpfs_clock_data *clock_data = dev_get_drvdata(dev->parent); - - return readl_relaxed(clock_data->base + REG_SUBBLK_RESET_CR); -} -EXPORT_SYMBOL_NS_GPL(mpfs_reset_read, MCHP_CLK_MPFS); - -void mpfs_reset_write(struct device *dev, u32 val) -{ - struct mpfs_clock_data *clock_data = dev_get_drvdata(dev->parent); - - writel_relaxed(val, clock_data->base + REG_SUBBLK_RESET_CR); -} -EXPORT_SYMBOL_NS_GPL(mpfs_reset_write, MCHP_CLK_MPFS); - -static void mpfs_reset_unregister_adev(void *_adev) -{ - struct auxiliary_device *adev = _adev; - - auxiliary_device_delete(adev); - auxiliary_device_uninit(adev); -} - -static void mpfs_reset_adev_release(struct device *dev) -{ - struct auxiliary_device *adev = to_auxiliary_dev(dev); - - kfree(adev); -} - -static struct auxiliary_device *mpfs_reset_adev_alloc(struct mpfs_clock_data *clk_data) -{ - struct auxiliary_device *adev; - int ret; - - adev = kzalloc(sizeof(*adev), GFP_KERNEL); - if (!adev) - return ERR_PTR(-ENOMEM); - - adev->name = "reset-mpfs"; - adev->dev.parent = clk_data->dev; - adev->dev.release = mpfs_reset_adev_release; - adev->id = 666u; - - ret = auxiliary_device_init(adev); - if (ret) { - kfree(adev); - return ERR_PTR(ret); - } - - return adev; -} - -static int mpfs_reset_controller_register(struct mpfs_clock_data *clk_data) -{ - struct auxiliary_device *adev; - int ret; - - adev = mpfs_reset_adev_alloc(clk_data); - if (IS_ERR(adev)) - return PTR_ERR(adev); - - ret = auxiliary_device_add(adev); - if (ret) { - auxiliary_device_uninit(adev); - return ret; - } - - return devm_add_action_or_reset(clk_data->dev, mpfs_reset_unregister_adev, adev); -} - -#else /* !CONFIG_RESET_CONTROLLER */ - -static int mpfs_reset_controller_register(struct mpfs_clock_data *clk_data) -{ - return 0; -} - -#endif /* !CONFIG_RESET_CONTROLLER */ - static int mpfs_clk_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -499,7 +410,7 @@ static int mpfs_clk_probe(struct platform_device *pdev) if (ret) return ret; - return mpfs_reset_controller_register(clk_data); + return mpfs_reset_controller_register(dev, clk_data->base + REG_SUBBLK_RESET_CR); } static const struct of_device_id mpfs_clk_of_match_table[] = { @@ -532,3 +443,4 @@ MODULE_DESCRIPTION("Microchip PolarFire SoC Clock Driver"); MODULE_AUTHOR("Padmarao Begari <padmarao.begari@microchip.com>"); MODULE_AUTHOR("Daire McNamara <daire.mcnamara@microchip.com>"); MODULE_AUTHOR("Conor Dooley <conor.dooley@microchip.com>"); +MODULE_IMPORT_NS(MCHP_CLK_MPFS); diff --git a/drivers/clk/nxp/clk-lpc18xx-cgu.c b/drivers/clk/nxp/clk-lpc18xx-cgu.c index 69ebf65081b8..81efa885069b 100644 --- a/drivers/clk/nxp/clk-lpc18xx-cgu.c +++ b/drivers/clk/nxp/clk-lpc18xx-cgu.c @@ -250,7 +250,6 @@ static struct lpc18xx_cgu_base_clk lpc18xx_cgu_base_clks[] = { struct lpc18xx_pll { struct clk_hw hw; void __iomem *reg; - spinlock_t *lock; u8 flags; }; diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index 8ab08e7b5b6c..1bb51a058872 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -474,6 +474,7 @@ config SC_CAMCC_7280 config SC_CAMCC_8280XP tristate "SC8280XP Camera Clock Controller" + depends on ARM64 || COMPILE_TEST select SC_GCC_8280XP help Support for the camera clock controller on Qualcomm Technologies, Inc @@ -1094,6 +1095,7 @@ config SM_GPUCC_8550 config SM_GPUCC_8650 tristate "SM8650 Graphics Clock Controller" + depends on ARM64 || COMPILE_TEST select SM_GCC_8650 help Support for the graphics clock controller on SM8650 devices. diff --git a/drivers/clk/qcom/apss-ipq-pll.c b/drivers/clk/qcom/apss-ipq-pll.c index 678b805f13d4..5f7f537e4ecb 100644 --- a/drivers/clk/qcom/apss-ipq-pll.c +++ b/drivers/clk/qcom/apss-ipq-pll.c @@ -8,61 +8,54 @@ #include "clk-alpha-pll.h" -/* - * Even though APSS PLL type is of existing one (like Huayra), its offsets - * are different from the one mentioned in the clk-alpha-pll.c, since the - * PLL is specific to APSS, so lets the define the same. - */ -static const u8 ipq_pll_offsets[][PLL_OFF_MAX_REGS] = { - [CLK_ALPHA_PLL_TYPE_HUAYRA] = { - [PLL_OFF_L_VAL] = 0x08, - [PLL_OFF_ALPHA_VAL] = 0x10, - [PLL_OFF_USER_CTL] = 0x18, - [PLL_OFF_CONFIG_CTL] = 0x20, - [PLL_OFF_CONFIG_CTL_U] = 0x24, - [PLL_OFF_STATUS] = 0x28, - [PLL_OFF_TEST_CTL] = 0x30, - [PLL_OFF_TEST_CTL_U] = 0x34, - }, - [CLK_ALPHA_PLL_TYPE_STROMER_PLUS] = { - [PLL_OFF_L_VAL] = 0x08, - [PLL_OFF_ALPHA_VAL] = 0x10, - [PLL_OFF_ALPHA_VAL_U] = 0x14, - [PLL_OFF_USER_CTL] = 0x18, - [PLL_OFF_USER_CTL_U] = 0x1c, - [PLL_OFF_CONFIG_CTL] = 0x20, - [PLL_OFF_STATUS] = 0x28, - [PLL_OFF_TEST_CTL] = 0x30, - [PLL_OFF_TEST_CTL_U] = 0x34, +static struct clk_alpha_pll ipq_pll_huayra = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_HUAYRA_APSS], + .flags = SUPPORTS_DYNAMIC_UPDATE, + .clkr = { + .enable_reg = 0x0, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "a53pll", + .parent_data = &(const struct clk_parent_data) { + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_huayra_ops, + }, }, }; -static struct clk_alpha_pll ipq_pll_huayra = { +static struct clk_alpha_pll ipq_pll_stromer = { .offset = 0x0, - .regs = ipq_pll_offsets[CLK_ALPHA_PLL_TYPE_HUAYRA], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_STROMER], .flags = SUPPORTS_DYNAMIC_UPDATE, .clkr = { .enable_reg = 0x0, .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ + .hw.init = &(const struct clk_init_data) { .name = "a53pll", .parent_data = &(const struct clk_parent_data) { .fw_name = "xo", }, .num_parents = 1, - .ops = &clk_alpha_pll_huayra_ops, + .ops = &clk_alpha_pll_stromer_ops, }, }, }; static struct clk_alpha_pll ipq_pll_stromer_plus = { .offset = 0x0, - .regs = ipq_pll_offsets[CLK_ALPHA_PLL_TYPE_STROMER_PLUS], + /* + * The register offsets of the Stromer Plus PLL used in IPQ5332 + * are the same as the Stromer PLL's offsets. + */ + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_STROMER], .flags = SUPPORTS_DYNAMIC_UPDATE, .clkr = { .enable_reg = 0x0, .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ + .hw.init = &(const struct clk_init_data) { .name = "a53pll", .parent_data = &(const struct clk_parent_data) { .fw_name = "xo", @@ -73,8 +66,9 @@ static struct clk_alpha_pll ipq_pll_stromer_plus = { }, }; +/* 1.008 GHz configuration */ static const struct alpha_pll_config ipq5018_pll_config = { - .l = 0x32, + .l = 0x2a, .config_ctl_val = 0x4001075b, .config_ctl_hi_val = 0x304, .main_output_mask = BIT(0), @@ -144,30 +138,30 @@ struct apss_pll_data { }; static const struct apss_pll_data ipq5018_pll_data = { - .pll_type = CLK_ALPHA_PLL_TYPE_STROMER_PLUS, - .pll = &ipq_pll_stromer_plus, + .pll_type = CLK_ALPHA_PLL_TYPE_STROMER, + .pll = &ipq_pll_stromer, .pll_config = &ipq5018_pll_config, }; -static struct apss_pll_data ipq5332_pll_data = { +static const struct apss_pll_data ipq5332_pll_data = { .pll_type = CLK_ALPHA_PLL_TYPE_STROMER_PLUS, .pll = &ipq_pll_stromer_plus, .pll_config = &ipq5332_pll_config, }; -static struct apss_pll_data ipq8074_pll_data = { +static const struct apss_pll_data ipq8074_pll_data = { .pll_type = CLK_ALPHA_PLL_TYPE_HUAYRA, .pll = &ipq_pll_huayra, .pll_config = &ipq8074_pll_config, }; -static struct apss_pll_data ipq6018_pll_data = { +static const struct apss_pll_data ipq6018_pll_data = { .pll_type = CLK_ALPHA_PLL_TYPE_HUAYRA, .pll = &ipq_pll_huayra, .pll_config = &ipq6018_pll_config, }; -static struct apss_pll_data ipq9574_pll_data = { +static const struct apss_pll_data ipq9574_pll_data = { .pll_type = CLK_ALPHA_PLL_TYPE_HUAYRA, .pll = &ipq_pll_huayra, .pll_config = &ipq9574_pll_config, @@ -203,7 +197,8 @@ static int apss_ipq_pll_probe(struct platform_device *pdev) if (data->pll_type == CLK_ALPHA_PLL_TYPE_HUAYRA) clk_alpha_pll_configure(data->pll, regmap, data->pll_config); - else if (data->pll_type == CLK_ALPHA_PLL_TYPE_STROMER_PLUS) + else if (data->pll_type == CLK_ALPHA_PLL_TYPE_STROMER || + data->pll_type == CLK_ALPHA_PLL_TYPE_STROMER_PLUS) clk_stromer_pll_configure(data->pll, regmap, data->pll_config); ret = devm_clk_register_regmap(dev, &data->pll->clkr); diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 8a412ef47e16..d4227909d1fe 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -83,6 +83,16 @@ const u8 clk_alpha_pll_regs[][PLL_OFF_MAX_REGS] = { [PLL_OFF_TEST_CTL_U] = 0x20, [PLL_OFF_STATUS] = 0x24, }, + [CLK_ALPHA_PLL_TYPE_HUAYRA_APSS] = { + [PLL_OFF_L_VAL] = 0x08, + [PLL_OFF_ALPHA_VAL] = 0x10, + [PLL_OFF_USER_CTL] = 0x18, + [PLL_OFF_CONFIG_CTL] = 0x20, + [PLL_OFF_CONFIG_CTL_U] = 0x24, + [PLL_OFF_STATUS] = 0x28, + [PLL_OFF_TEST_CTL] = 0x30, + [PLL_OFF_TEST_CTL_U] = 0x34, + }, [CLK_ALPHA_PLL_TYPE_BRAMMO] = { [PLL_OFF_L_VAL] = 0x04, [PLL_OFF_ALPHA_VAL] = 0x08, @@ -213,10 +223,9 @@ const u8 clk_alpha_pll_regs[][PLL_OFF_MAX_REGS] = { [PLL_OFF_USER_CTL] = 0x18, [PLL_OFF_USER_CTL_U] = 0x1c, [PLL_OFF_CONFIG_CTL] = 0x20, - [PLL_OFF_CONFIG_CTL_U] = 0xff, + [PLL_OFF_STATUS] = 0x28, [PLL_OFF_TEST_CTL] = 0x30, [PLL_OFF_TEST_CTL_U] = 0x34, - [PLL_OFF_STATUS] = 0x28, }, [CLK_ALPHA_PLL_TYPE_STROMER_PLUS] = { [PLL_OFF_L_VAL] = 0x04, @@ -2114,6 +2123,15 @@ void clk_lucid_evo_pll_configure(struct clk_alpha_pll *pll, struct regmap *regma { u32 lval = config->l; + /* + * If the bootloader left the PLL enabled it's likely that there are + * RCGs that will lock up if we disable the PLL below. + */ + if (trion_pll_is_enabled(pll, regmap)) { + pr_debug("Lucid Evo PLL is already enabled, skipping configuration\n"); + return; + } + lval |= TRION_PLL_CAL_VAL << LUCID_EVO_PLL_CAL_L_VAL_SHIFT; clk_alpha_pll_write_config(regmap, PLL_L_VAL(pll), lval); clk_alpha_pll_write_config(regmap, PLL_ALPHA_VAL(pll), config->alpha); @@ -2490,6 +2508,8 @@ static int clk_alpha_pll_stromer_set_rate(struct clk_hw *hw, unsigned long rate, rate = alpha_pll_round_rate(rate, prate, &l, &a, ALPHA_REG_BITWIDTH); regmap_write(pll->clkr.regmap, PLL_L_VAL(pll), l); + + a <<= ALPHA_REG_BITWIDTH - ALPHA_BITWIDTH; regmap_write(pll->clkr.regmap, PLL_ALPHA_VAL(pll), a); regmap_write(pll->clkr.regmap, PLL_ALPHA_VAL_U(pll), a >> ALPHA_BITWIDTH); diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h index fb6d50263bb9..c7055b6c42f1 100644 --- a/drivers/clk/qcom/clk-alpha-pll.h +++ b/drivers/clk/qcom/clk-alpha-pll.h @@ -15,6 +15,7 @@ enum { CLK_ALPHA_PLL_TYPE_DEFAULT, CLK_ALPHA_PLL_TYPE_HUAYRA, + CLK_ALPHA_PLL_TYPE_HUAYRA_APSS, CLK_ALPHA_PLL_TYPE_BRAMMO, CLK_ALPHA_PLL_TYPE_FABIA, CLK_ALPHA_PLL_TYPE_TRION, @@ -73,8 +74,10 @@ struct pll_vco { /** * struct clk_alpha_pll - phase locked loop (PLL) * @offset: base address of registers - * @vco_table: array of VCO settings * @regs: alpha pll register map (see @clk_alpha_pll_regs) + * @vco_table: array of VCO settings + * @num_vco: number of VCO settings in @vco_table + * @flags: bitmask to indicate features supported by the hardware * @clkr: regmap clock handle */ struct clk_alpha_pll { diff --git a/drivers/clk/qcom/clk-cbf-8996.c b/drivers/clk/qcom/clk-cbf-8996.c index fe24b4abeab4..76bf523431b8 100644 --- a/drivers/clk/qcom/clk-cbf-8996.c +++ b/drivers/clk/qcom/clk-cbf-8996.c @@ -41,17 +41,6 @@ enum { #define CBF_PLL_OFFSET 0xf000 -static const u8 cbf_pll_regs[PLL_OFF_MAX_REGS] = { - [PLL_OFF_L_VAL] = 0x08, - [PLL_OFF_ALPHA_VAL] = 0x10, - [PLL_OFF_USER_CTL] = 0x18, - [PLL_OFF_CONFIG_CTL] = 0x20, - [PLL_OFF_CONFIG_CTL_U] = 0x24, - [PLL_OFF_TEST_CTL] = 0x30, - [PLL_OFF_TEST_CTL_U] = 0x34, - [PLL_OFF_STATUS] = 0x28, -}; - static struct alpha_pll_config cbfpll_config = { .l = 72, .config_ctl_val = 0x200d4828, @@ -67,7 +56,7 @@ static struct alpha_pll_config cbfpll_config = { static struct clk_alpha_pll cbf_pll = { .offset = CBF_PLL_OFFSET, - .regs = cbf_pll_regs, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_HUAYRA_APSS], .flags = SUPPORTS_DYNAMIC_UPDATE | SUPPORTS_FSM_MODE, .clkr.hw.init = &(struct clk_init_data){ .name = "cbf_pll", diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h index e6d84c8c7989..d7414361e432 100644 --- a/drivers/clk/qcom/clk-rcg.h +++ b/drivers/clk/qcom/clk-rcg.h @@ -17,6 +17,23 @@ struct freq_tbl { u16 n; }; +#define C(s, h, m, n) { (s), (2 * (h) - 1), (m), (n) } +#define FM(f, confs) { (f), ARRAY_SIZE(confs), (confs) } +#define FMS(f, s, h, m, n) { (f), 1, (const struct freq_conf []){ C(s, h, m, n) } } + +struct freq_conf { + u8 src; + u8 pre_div; + u16 m; + u16 n; +}; + +struct freq_multi_tbl { + unsigned long freq; + size_t num_confs; + const struct freq_conf *confs; +}; + /** * struct mn - M/N:D counter * @mnctr_en_bit: bit to enable mn counter @@ -138,6 +155,7 @@ extern const struct clk_ops clk_dyn_rcg_ops; * @safe_src_index: safe src index value * @parent_map: map from software's parent index to hardware's src_sel field * @freq_tbl: frequency table + * @freq_multi_tbl: frequency table for clocks reachable with multiple RCGs conf * @clkr: regmap clock handle * @cfg_off: defines the cfg register offset from the CMD_RCGR + CFG_REG * @parked_cfg: cached value of the CFG register for parked RCGs @@ -149,7 +167,10 @@ struct clk_rcg2 { u8 hid_width; u8 safe_src_index; const struct parent_map *parent_map; - const struct freq_tbl *freq_tbl; + union { + const struct freq_tbl *freq_tbl; + const struct freq_multi_tbl *freq_multi_tbl; + }; struct clk_regmap clkr; u8 cfg_off; u32 parked_cfg; @@ -169,6 +190,7 @@ struct clk_rcg2_gfx3d { extern const struct clk_ops clk_rcg2_ops; extern const struct clk_ops clk_rcg2_floor_ops; +extern const struct clk_ops clk_rcg2_fm_ops; extern const struct clk_ops clk_rcg2_mux_closest_ops; extern const struct clk_ops clk_edp_pixel_ops; extern const struct clk_ops clk_byte_ops; diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index 5183c74b074f..9b3aaa7f20ac 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -260,6 +260,115 @@ static int _freq_tbl_determine_rate(struct clk_hw *hw, const struct freq_tbl *f, return 0; } +static const struct freq_conf * +__clk_rcg2_select_conf(struct clk_hw *hw, const struct freq_multi_tbl *f, + unsigned long req_rate) +{ + unsigned long rate_diff, best_rate_diff = ULONG_MAX; + const struct freq_conf *conf, *best_conf = NULL; + struct clk_rcg2 *rcg = to_clk_rcg2(hw); + const char *name = clk_hw_get_name(hw); + unsigned long parent_rate, rate; + struct clk_hw *p; + int index, i; + + /* Exit early if only one config is defined */ + if (f->num_confs == 1) { + best_conf = f->confs; + goto exit; + } + + /* Search in each provided config the one that is near the wanted rate */ + for (i = 0, conf = f->confs; i < f->num_confs; i++, conf++) { + index = qcom_find_src_index(hw, rcg->parent_map, conf->src); + if (index < 0) + continue; + + p = clk_hw_get_parent_by_index(hw, index); + if (!p) + continue; + + parent_rate = clk_hw_get_rate(p); + rate = calc_rate(parent_rate, conf->n, conf->m, conf->n, conf->pre_div); + + if (rate == req_rate) { + best_conf = conf; + goto exit; + } + + rate_diff = abs_diff(req_rate, rate); + if (rate_diff < best_rate_diff) { + best_rate_diff = rate_diff; + best_conf = conf; + } + } + + /* + * Very unlikely. Warn if we couldn't find a correct config + * due to parent not found in every config. + */ + if (unlikely(!best_conf)) { + WARN(1, "%s: can't find a configuration for rate %lu\n", + name, req_rate); + return ERR_PTR(-EINVAL); + } + +exit: + return best_conf; +} + +static int _freq_tbl_fm_determine_rate(struct clk_hw *hw, const struct freq_multi_tbl *f, + struct clk_rate_request *req) +{ + unsigned long clk_flags, rate = req->rate; + struct clk_rcg2 *rcg = to_clk_rcg2(hw); + const struct freq_conf *conf; + struct clk_hw *p; + int index; + + f = qcom_find_freq_multi(f, rate); + if (!f || !f->confs) + return -EINVAL; + + conf = __clk_rcg2_select_conf(hw, f, rate); + if (IS_ERR(conf)) + return PTR_ERR(conf); + index = qcom_find_src_index(hw, rcg->parent_map, conf->src); + if (index < 0) + return index; + + clk_flags = clk_hw_get_flags(hw); + p = clk_hw_get_parent_by_index(hw, index); + if (!p) + return -EINVAL; + + if (clk_flags & CLK_SET_RATE_PARENT) { + rate = f->freq; + if (conf->pre_div) { + if (!rate) + rate = req->rate; + rate /= 2; + rate *= conf->pre_div + 1; + } + + if (conf->n) { + u64 tmp = rate; + + tmp = tmp * conf->n; + do_div(tmp, conf->m); + rate = tmp; + } + } else { + rate = clk_hw_get_rate(p); + } + + req->best_parent_hw = p; + req->best_parent_rate = rate; + req->rate = f->freq; + + return 0; +} + static int clk_rcg2_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { @@ -276,6 +385,14 @@ static int clk_rcg2_determine_floor_rate(struct clk_hw *hw, return _freq_tbl_determine_rate(hw, rcg->freq_tbl, req, FLOOR); } +static int clk_rcg2_fm_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct clk_rcg2 *rcg = to_clk_rcg2(hw); + + return _freq_tbl_fm_determine_rate(hw, rcg->freq_multi_tbl, req); +} + static int __clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f, u32 *_cfg) { @@ -371,6 +488,30 @@ static int __clk_rcg2_set_rate(struct clk_hw *hw, unsigned long rate, return clk_rcg2_configure(rcg, f); } +static int __clk_rcg2_fm_set_rate(struct clk_hw *hw, unsigned long rate) +{ + struct clk_rcg2 *rcg = to_clk_rcg2(hw); + const struct freq_multi_tbl *f; + const struct freq_conf *conf; + struct freq_tbl f_tbl = {}; + + f = qcom_find_freq_multi(rcg->freq_multi_tbl, rate); + if (!f || !f->confs) + return -EINVAL; + + conf = __clk_rcg2_select_conf(hw, f, rate); + if (IS_ERR(conf)) + return PTR_ERR(conf); + + f_tbl.freq = f->freq; + f_tbl.src = conf->src; + f_tbl.pre_div = conf->pre_div; + f_tbl.m = conf->m; + f_tbl.n = conf->n; + + return clk_rcg2_configure(rcg, &f_tbl); +} + static int clk_rcg2_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { @@ -383,6 +524,12 @@ static int clk_rcg2_set_floor_rate(struct clk_hw *hw, unsigned long rate, return __clk_rcg2_set_rate(hw, rate, FLOOR); } +static int clk_rcg2_fm_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + return __clk_rcg2_fm_set_rate(hw, rate); +} + static int clk_rcg2_set_rate_and_parent(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate, u8 index) { @@ -395,6 +542,12 @@ static int clk_rcg2_set_floor_rate_and_parent(struct clk_hw *hw, return __clk_rcg2_set_rate(hw, rate, FLOOR); } +static int clk_rcg2_fm_set_rate_and_parent(struct clk_hw *hw, + unsigned long rate, unsigned long parent_rate, u8 index) +{ + return __clk_rcg2_fm_set_rate(hw, rate); +} + static int clk_rcg2_get_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) { struct clk_rcg2 *rcg = to_clk_rcg2(hw); @@ -505,6 +658,19 @@ const struct clk_ops clk_rcg2_floor_ops = { }; EXPORT_SYMBOL_GPL(clk_rcg2_floor_ops); +const struct clk_ops clk_rcg2_fm_ops = { + .is_enabled = clk_rcg2_is_enabled, + .get_parent = clk_rcg2_get_parent, + .set_parent = clk_rcg2_set_parent, + .recalc_rate = clk_rcg2_recalc_rate, + .determine_rate = clk_rcg2_fm_determine_rate, + .set_rate = clk_rcg2_fm_set_rate, + .set_rate_and_parent = clk_rcg2_fm_set_rate_and_parent, + .get_duty_cycle = clk_rcg2_get_duty_cycle, + .set_duty_cycle = clk_rcg2_set_duty_cycle, +}; +EXPORT_SYMBOL_GPL(clk_rcg2_fm_ops); + const struct clk_ops clk_rcg2_mux_closest_ops = { .determine_rate = __clk_mux_determine_rate_closest, .get_parent = clk_rcg2_get_parent, diff --git a/drivers/clk/qcom/clk-rpm.c b/drivers/clk/qcom/clk-rpm.c index 745026ef4d9c..9da034f8f2ff 100644 --- a/drivers/clk/qcom/clk-rpm.c +++ b/drivers/clk/qcom/clk-rpm.c @@ -98,7 +98,6 @@ struct clk_rpm { }; struct rpm_cc { - struct qcom_rpm *rpm; struct clk_rpm **clks; size_t num_clks; u32 xo_buffer_value; diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index 75f09e6e057e..48f81e3a5e80 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -41,6 +41,24 @@ struct freq_tbl *qcom_find_freq(const struct freq_tbl *f, unsigned long rate) } EXPORT_SYMBOL_GPL(qcom_find_freq); +const struct freq_multi_tbl *qcom_find_freq_multi(const struct freq_multi_tbl *f, + unsigned long rate) +{ + if (!f) + return NULL; + + if (!f->freq) + return f; + + for (; f->freq; f++) + if (rate <= f->freq) + return f; + + /* Default to our fastest rate */ + return f - 1; +} +EXPORT_SYMBOL_GPL(qcom_find_freq_multi); + const struct freq_tbl *qcom_find_freq_floor(const struct freq_tbl *f, unsigned long rate) { diff --git a/drivers/clk/qcom/common.h b/drivers/clk/qcom/common.h index 9c8f7b798d9f..2d4a8a837e6c 100644 --- a/drivers/clk/qcom/common.h +++ b/drivers/clk/qcom/common.h @@ -45,6 +45,8 @@ extern const struct freq_tbl *qcom_find_freq(const struct freq_tbl *f, unsigned long rate); extern const struct freq_tbl *qcom_find_freq_floor(const struct freq_tbl *f, unsigned long rate); +extern const struct freq_multi_tbl *qcom_find_freq_multi(const struct freq_multi_tbl *f, + unsigned long rate); extern void qcom_pll_set_fsm_mode(struct regmap *m, u32 reg, u8 bias_count, u8 lock_count); extern int qcom_find_src_index(struct clk_hw *hw, const struct parent_map *map, diff --git a/drivers/clk/qcom/dispcc-sm6350.c b/drivers/clk/qcom/dispcc-sm6350.c index 839435362010..e4b7464c4d0e 100644 --- a/drivers/clk/qcom/dispcc-sm6350.c +++ b/drivers/clk/qcom/dispcc-sm6350.c @@ -221,26 +221,17 @@ static struct clk_rcg2 disp_cc_mdss_dp_crypto_clk_src = { }, }; -static const struct freq_tbl ftbl_disp_cc_mdss_dp_link_clk_src[] = { - F(162000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), - F(270000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), - F(540000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), - F(810000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), - { } -}; - static struct clk_rcg2 disp_cc_mdss_dp_link_clk_src = { .cmd_rcgr = 0x10f8, .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_0, - .freq_tbl = ftbl_disp_cc_mdss_dp_link_clk_src, .clkr.hw.init = &(struct clk_init_data){ .name = "disp_cc_mdss_dp_link_clk_src", .parent_data = disp_cc_parent_data_0, .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm8450.c b/drivers/clk/qcom/dispcc-sm8450.c index 92e9c4e7b13d..49bb4f58c391 100644 --- a/drivers/clk/qcom/dispcc-sm8450.c +++ b/drivers/clk/qcom/dispcc-sm8450.c @@ -309,26 +309,17 @@ static struct clk_rcg2 disp_cc_mdss_dptx0_aux_clk_src = { }, }; -static const struct freq_tbl ftbl_disp_cc_mdss_dptx0_link_clk_src[] = { - F(162000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(270000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(540000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(810000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - { } -}; - static struct clk_rcg2 disp_cc_mdss_dptx0_link_clk_src = { .cmd_rcgr = 0x819c, .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx0_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -382,13 +373,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx1_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx1_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -442,13 +432,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx2_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx2_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -502,13 +491,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx3_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx3_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm8550.c b/drivers/clk/qcom/dispcc-sm8550.c index 3672c73ac11c..38ecea805503 100644 --- a/drivers/clk/qcom/dispcc-sm8550.c +++ b/drivers/clk/qcom/dispcc-sm8550.c @@ -345,26 +345,17 @@ static struct clk_rcg2 disp_cc_mdss_dptx0_aux_clk_src = { }, }; -static const struct freq_tbl ftbl_disp_cc_mdss_dptx0_link_clk_src[] = { - F(162000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(270000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(540000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(810000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - { } -}; - static struct clk_rcg2 disp_cc_mdss_dptx0_link_clk_src = { .cmd_rcgr = 0x8170, .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_7, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx0_link_clk_src", .parent_data = disp_cc_parent_data_7, .num_parents = ARRAY_SIZE(disp_cc_parent_data_7), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -418,13 +409,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx1_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx1_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -478,13 +468,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx2_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx2_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -538,13 +527,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx3_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx3_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm8650.c b/drivers/clk/qcom/dispcc-sm8650.c index 9539db0d9114..3eb64bcad487 100644 --- a/drivers/clk/qcom/dispcc-sm8650.c +++ b/drivers/clk/qcom/dispcc-sm8650.c @@ -343,26 +343,17 @@ static struct clk_rcg2 disp_cc_mdss_dptx0_aux_clk_src = { }, }; -static const struct freq_tbl ftbl_disp_cc_mdss_dptx0_link_clk_src[] = { - F(162000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(270000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(540000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(810000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - { } -}; - static struct clk_rcg2 disp_cc_mdss_dptx0_link_clk_src = { .cmd_rcgr = 0x8170, .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_7, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(const struct clk_init_data) { .name = "disp_cc_mdss_dptx0_link_clk_src", .parent_data = disp_cc_parent_data_7, .num_parents = ARRAY_SIZE(disp_cc_parent_data_7), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -416,13 +407,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx1_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(const struct clk_init_data) { .name = "disp_cc_mdss_dptx1_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -476,13 +466,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx2_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(const struct clk_init_data) { .name = "disp_cc_mdss_dptx2_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -536,13 +525,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx3_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(const struct clk_init_data) { .name = "disp_cc_mdss_dptx3_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; diff --git a/drivers/clk/qcom/gcc-ipq8074.c b/drivers/clk/qcom/gcc-ipq8074.c index 7bc679871f32..d2be56c5892d 100644 --- a/drivers/clk/qcom/gcc-ipq8074.c +++ b/drivers/clk/qcom/gcc-ipq8074.c @@ -1677,15 +1677,23 @@ static struct clk_regmap_div nss_port4_tx_div_clk_src = { }, }; -static const struct freq_tbl ftbl_nss_port5_rx_clk_src[] = { - F(19200000, P_XO, 1, 0, 0), - F(25000000, P_UNIPHY1_RX, 12.5, 0, 0), - F(25000000, P_UNIPHY0_RX, 5, 0, 0), - F(78125000, P_UNIPHY1_RX, 4, 0, 0), - F(125000000, P_UNIPHY1_RX, 2.5, 0, 0), - F(125000000, P_UNIPHY0_RX, 1, 0, 0), - F(156250000, P_UNIPHY1_RX, 2, 0, 0), - F(312500000, P_UNIPHY1_RX, 1, 0, 0), +static const struct freq_conf ftbl_nss_port5_rx_clk_src_25[] = { + C(P_UNIPHY1_RX, 12.5, 0, 0), + C(P_UNIPHY0_RX, 5, 0, 0), +}; + +static const struct freq_conf ftbl_nss_port5_rx_clk_src_125[] = { + C(P_UNIPHY1_RX, 2.5, 0, 0), + C(P_UNIPHY0_RX, 1, 0, 0), +}; + +static const struct freq_multi_tbl ftbl_nss_port5_rx_clk_src[] = { + FMS(19200000, P_XO, 1, 0, 0), + FM(25000000, ftbl_nss_port5_rx_clk_src_25), + FMS(78125000, P_UNIPHY1_RX, 4, 0, 0), + FM(125000000, ftbl_nss_port5_rx_clk_src_125), + FMS(156250000, P_UNIPHY1_RX, 2, 0, 0), + FMS(312500000, P_UNIPHY1_RX, 1, 0, 0), { } }; @@ -1712,14 +1720,14 @@ gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias_map[] = { static struct clk_rcg2 nss_port5_rx_clk_src = { .cmd_rcgr = 0x68060, - .freq_tbl = ftbl_nss_port5_rx_clk_src, + .freq_multi_tbl = ftbl_nss_port5_rx_clk_src, .hid_width = 5, .parent_map = gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias_map, .clkr.hw.init = &(struct clk_init_data){ .name = "nss_port5_rx_clk_src", .parent_data = gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias, .num_parents = ARRAY_SIZE(gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias), - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_fm_ops, }, }; @@ -1739,15 +1747,23 @@ static struct clk_regmap_div nss_port5_rx_div_clk_src = { }, }; -static const struct freq_tbl ftbl_nss_port5_tx_clk_src[] = { - F(19200000, P_XO, 1, 0, 0), - F(25000000, P_UNIPHY1_TX, 12.5, 0, 0), - F(25000000, P_UNIPHY0_TX, 5, 0, 0), - F(78125000, P_UNIPHY1_TX, 4, 0, 0), - F(125000000, P_UNIPHY1_TX, 2.5, 0, 0), - F(125000000, P_UNIPHY0_TX, 1, 0, 0), - F(156250000, P_UNIPHY1_TX, 2, 0, 0), - F(312500000, P_UNIPHY1_TX, 1, 0, 0), +static const struct freq_conf ftbl_nss_port5_tx_clk_src_25[] = { + C(P_UNIPHY1_TX, 12.5, 0, 0), + C(P_UNIPHY0_TX, 5, 0, 0), +}; + +static const struct freq_conf ftbl_nss_port5_tx_clk_src_125[] = { + C(P_UNIPHY1_TX, 2.5, 0, 0), + C(P_UNIPHY0_TX, 1, 0, 0), +}; + +static const struct freq_multi_tbl ftbl_nss_port5_tx_clk_src[] = { + FMS(19200000, P_XO, 1, 0, 0), + FM(25000000, ftbl_nss_port5_tx_clk_src_25), + FMS(78125000, P_UNIPHY1_TX, 4, 0, 0), + FM(125000000, ftbl_nss_port5_tx_clk_src_125), + FMS(156250000, P_UNIPHY1_TX, 2, 0, 0), + FMS(312500000, P_UNIPHY1_TX, 1, 0, 0), { } }; @@ -1774,14 +1790,14 @@ gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias_map[] = { static struct clk_rcg2 nss_port5_tx_clk_src = { .cmd_rcgr = 0x68068, - .freq_tbl = ftbl_nss_port5_tx_clk_src, + .freq_multi_tbl = ftbl_nss_port5_tx_clk_src, .hid_width = 5, .parent_map = gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias_map, .clkr.hw.init = &(struct clk_init_data){ .name = "nss_port5_tx_clk_src", .parent_data = gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias, .num_parents = ARRAY_SIZE(gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias), - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_fm_ops, }, }; @@ -1801,15 +1817,23 @@ static struct clk_regmap_div nss_port5_tx_div_clk_src = { }, }; -static const struct freq_tbl ftbl_nss_port6_rx_clk_src[] = { - F(19200000, P_XO, 1, 0, 0), - F(25000000, P_UNIPHY2_RX, 5, 0, 0), - F(25000000, P_UNIPHY2_RX, 12.5, 0, 0), - F(78125000, P_UNIPHY2_RX, 4, 0, 0), - F(125000000, P_UNIPHY2_RX, 1, 0, 0), - F(125000000, P_UNIPHY2_RX, 2.5, 0, 0), - F(156250000, P_UNIPHY2_RX, 2, 0, 0), - F(312500000, P_UNIPHY2_RX, 1, 0, 0), +static const struct freq_conf ftbl_nss_port6_rx_clk_src_25[] = { + C(P_UNIPHY2_RX, 5, 0, 0), + C(P_UNIPHY2_RX, 12.5, 0, 0), +}; + +static const struct freq_conf ftbl_nss_port6_rx_clk_src_125[] = { + C(P_UNIPHY2_RX, 1, 0, 0), + C(P_UNIPHY2_RX, 2.5, 0, 0), +}; + +static const struct freq_multi_tbl ftbl_nss_port6_rx_clk_src[] = { + FMS(19200000, P_XO, 1, 0, 0), + FM(25000000, ftbl_nss_port6_rx_clk_src_25), + FMS(78125000, P_UNIPHY2_RX, 4, 0, 0), + FM(125000000, ftbl_nss_port6_rx_clk_src_125), + FMS(156250000, P_UNIPHY2_RX, 2, 0, 0), + FMS(312500000, P_UNIPHY2_RX, 1, 0, 0), { } }; @@ -1831,14 +1855,14 @@ static const struct parent_map gcc_xo_uniphy2_rx_tx_ubi32_bias_map[] = { static struct clk_rcg2 nss_port6_rx_clk_src = { .cmd_rcgr = 0x68070, - .freq_tbl = ftbl_nss_port6_rx_clk_src, + .freq_multi_tbl = ftbl_nss_port6_rx_clk_src, .hid_width = 5, .parent_map = gcc_xo_uniphy2_rx_tx_ubi32_bias_map, .clkr.hw.init = &(struct clk_init_data){ .name = "nss_port6_rx_clk_src", .parent_data = gcc_xo_uniphy2_rx_tx_ubi32_bias, .num_parents = ARRAY_SIZE(gcc_xo_uniphy2_rx_tx_ubi32_bias), - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_fm_ops, }, }; @@ -1858,15 +1882,23 @@ static struct clk_regmap_div nss_port6_rx_div_clk_src = { }, }; -static const struct freq_tbl ftbl_nss_port6_tx_clk_src[] = { - F(19200000, P_XO, 1, 0, 0), - F(25000000, P_UNIPHY2_TX, 5, 0, 0), - F(25000000, P_UNIPHY2_TX, 12.5, 0, 0), - F(78125000, P_UNIPHY2_TX, 4, 0, 0), - F(125000000, P_UNIPHY2_TX, 1, 0, 0), - F(125000000, P_UNIPHY2_TX, 2.5, 0, 0), - F(156250000, P_UNIPHY2_TX, 2, 0, 0), - F(312500000, P_UNIPHY2_TX, 1, 0, 0), +static const struct freq_conf ftbl_nss_port6_tx_clk_src_25[] = { + C(P_UNIPHY2_TX, 5, 0, 0), + C(P_UNIPHY2_TX, 12.5, 0, 0), +}; + +static const struct freq_conf ftbl_nss_port6_tx_clk_src_125[] = { + C(P_UNIPHY2_TX, 1, 0, 0), + C(P_UNIPHY2_TX, 2.5, 0, 0), +}; + +static const struct freq_multi_tbl ftbl_nss_port6_tx_clk_src[] = { + FMS(19200000, P_XO, 1, 0, 0), + FM(25000000, ftbl_nss_port6_tx_clk_src_25), + FMS(78125000, P_UNIPHY1_RX, 4, 0, 0), + FM(125000000, ftbl_nss_port6_tx_clk_src_125), + FMS(156250000, P_UNIPHY1_RX, 2, 0, 0), + FMS(312500000, P_UNIPHY1_RX, 1, 0, 0), { } }; @@ -1888,14 +1920,14 @@ static const struct parent_map gcc_xo_uniphy2_tx_rx_ubi32_bias_map[] = { static struct clk_rcg2 nss_port6_tx_clk_src = { .cmd_rcgr = 0x68078, - .freq_tbl = ftbl_nss_port6_tx_clk_src, + .freq_multi_tbl = ftbl_nss_port6_tx_clk_src, .hid_width = 5, .parent_map = gcc_xo_uniphy2_tx_rx_ubi32_bias_map, .clkr.hw.init = &(struct clk_init_data){ .name = "nss_port6_tx_clk_src", .parent_data = gcc_xo_uniphy2_tx_rx_ubi32_bias, .num_parents = ARRAY_SIZE(gcc_xo_uniphy2_tx_rx_ubi32_bias), - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_fm_ops, }, }; diff --git a/drivers/clk/qcom/gcc-msm8917.c b/drivers/clk/qcom/gcc-msm8917.c index f2dd132e2fb1..f2b8729e4198 100644 --- a/drivers/clk/qcom/gcc-msm8917.c +++ b/drivers/clk/qcom/gcc-msm8917.c @@ -3278,6 +3278,7 @@ static const struct of_device_id gcc_msm8917_match_table[] = { { .compatible = "qcom,gcc-qm215", .data = &gcc_qm215_desc }, {}, }; +MODULE_DEVICE_TABLE(of, gcc_msm8917_match_table); static struct platform_driver gcc_msm8917_driver = { .probe = gcc_msm8917_probe, diff --git a/drivers/clk/qcom/gcc-msm8953.c b/drivers/clk/qcom/gcc-msm8953.c index 68359534ff25..7563bff58118 100644 --- a/drivers/clk/qcom/gcc-msm8953.c +++ b/drivers/clk/qcom/gcc-msm8953.c @@ -4227,6 +4227,7 @@ static const struct of_device_id gcc_msm8953_match_table[] = { { .compatible = "qcom,gcc-msm8953" }, {}, }; +MODULE_DEVICE_TABLE(of, gcc_msm8953_match_table); static struct platform_driver gcc_msm8953_driver = { .probe = gcc_msm8953_probe, diff --git a/drivers/clk/qcom/gcc-sm8150.c b/drivers/clk/qcom/gcc-sm8150.c index a47ef9dfa808..1f748141d12c 100644 --- a/drivers/clk/qcom/gcc-sm8150.c +++ b/drivers/clk/qcom/gcc-sm8150.c @@ -207,28 +207,6 @@ static const struct clk_parent_data gcc_parents_7[] = { { .hw = &gpll0_out_even.clkr.hw }, }; -static const struct freq_tbl ftbl_gcc_cpuss_ahb_clk_src[] = { - F(19200000, P_BI_TCXO, 1, 0, 0), - F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), - F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), - { } -}; - -static struct clk_rcg2 gcc_cpuss_ahb_clk_src = { - .cmd_rcgr = 0x48014, - .mnd_width = 0, - .hid_width = 5, - .parent_map = gcc_parent_map_0, - .freq_tbl = ftbl_gcc_cpuss_ahb_clk_src, - .clkr.hw.init = &(struct clk_init_data){ - .name = "gcc_cpuss_ahb_clk_src", - .parent_data = gcc_parents_0, - .num_parents = ARRAY_SIZE(gcc_parents_0), - .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, - }, -}; - static const struct freq_tbl ftbl_gcc_emac_ptp_clk_src[] = { F(19200000, P_BI_TCXO, 1, 0, 0), F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), @@ -1361,24 +1339,6 @@ static struct clk_branch gcc_cfg_noc_usb3_sec_axi_clk = { }, }; -static struct clk_branch gcc_cpuss_ahb_clk = { - .halt_reg = 0x48000, - .halt_check = BRANCH_HALT_VOTED, - .clkr = { - .enable_reg = 0x52004, - .enable_mask = BIT(21), - .hw.init = &(struct clk_init_data){ - .name = "gcc_cpuss_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ - &gcc_cpuss_ahb_clk_src.clkr.hw }, - .num_parents = 1, - /* required for cpuss */ - .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_cpuss_dvm_bus_clk = { .halt_reg = 0x48190, .halt_check = BRANCH_HALT, @@ -2685,24 +2645,6 @@ static struct clk_branch gcc_sdcc4_apps_clk = { }, }; -static struct clk_branch gcc_sys_noc_cpuss_ahb_clk = { - .halt_reg = 0x4819c, - .halt_check = BRANCH_HALT_VOTED, - .clkr = { - .enable_reg = 0x52004, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gcc_sys_noc_cpuss_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ - &gcc_cpuss_ahb_clk_src.clkr.hw }, - .num_parents = 1, - /* required for cpuss */ - .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_tsif_ahb_clk = { .halt_reg = 0x36004, .halt_check = BRANCH_HALT, @@ -3550,8 +3492,6 @@ static struct clk_regmap *gcc_sm8150_clocks[] = { [GCC_CAMERA_XO_CLK] = &gcc_camera_xo_clk.clkr, [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, [GCC_CFG_NOC_USB3_SEC_AXI_CLK] = &gcc_cfg_noc_usb3_sec_axi_clk.clkr, - [GCC_CPUSS_AHB_CLK] = &gcc_cpuss_ahb_clk.clkr, - [GCC_CPUSS_AHB_CLK_SRC] = &gcc_cpuss_ahb_clk_src.clkr, [GCC_CPUSS_DVM_BUS_CLK] = &gcc_cpuss_dvm_bus_clk.clkr, [GCC_CPUSS_GNOC_CLK] = &gcc_cpuss_gnoc_clk.clkr, [GCC_CPUSS_RBCPR_CLK] = &gcc_cpuss_rbcpr_clk.clkr, @@ -3669,7 +3609,6 @@ static struct clk_regmap *gcc_sm8150_clocks[] = { [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr, - [GCC_SYS_NOC_CPUSS_AHB_CLK] = &gcc_sys_noc_cpuss_ahb_clk.clkr, [GCC_TSIF_AHB_CLK] = &gcc_tsif_ahb_clk.clkr, [GCC_TSIF_INACTIVITY_TIMERS_CLK] = &gcc_tsif_inactivity_timers_clk.clkr, [GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr, diff --git a/drivers/clk/qcom/hfpll.c b/drivers/clk/qcom/hfpll.c index dac27e31ef60..b0b0cb074b4a 100644 --- a/drivers/clk/qcom/hfpll.c +++ b/drivers/clk/qcom/hfpll.c @@ -14,7 +14,7 @@ #include "clk-regmap.h" #include "clk-hfpll.h" -static const struct hfpll_data hdata = { +static const struct hfpll_data qcs404 = { .mode_reg = 0x00, .l_reg = 0x04, .m_reg = 0x08, @@ -84,10 +84,12 @@ static const struct hfpll_data msm8976_cci = { }; static const struct of_device_id qcom_hfpll_match_table[] = { - { .compatible = "qcom,hfpll", .data = &hdata }, { .compatible = "qcom,msm8976-hfpll-a53", .data = &msm8976_a53 }, { .compatible = "qcom,msm8976-hfpll-a72", .data = &msm8976_a72 }, { .compatible = "qcom,msm8976-hfpll-cci", .data = &msm8976_cci }, + { .compatible = "qcom,qcs404-hfpll", .data = &qcs404 }, + /* Deprecated in bindings */ + { .compatible = "qcom,hfpll", .data = &qcs404 }, { } }; MODULE_DEVICE_TABLE(of, qcom_hfpll_match_table); diff --git a/drivers/clk/qcom/mmcc-msm8998.c b/drivers/clk/qcom/mmcc-msm8998.c index 1180e48c687a..275fb3b71ede 100644 --- a/drivers/clk/qcom/mmcc-msm8998.c +++ b/drivers/clk/qcom/mmcc-msm8998.c @@ -2535,6 +2535,8 @@ static struct clk_branch vmem_ahb_clk = { static struct gdsc video_top_gdsc = { .gdscr = 0x1024, + .cxcs = (unsigned int []){ 0x1028, 0x1034, 0x1038 }, + .cxc_count = 3, .pd = { .name = "video_top", }, @@ -2543,20 +2545,26 @@ static struct gdsc video_top_gdsc = { static struct gdsc video_subcore0_gdsc = { .gdscr = 0x1040, + .cxcs = (unsigned int []){ 0x1048 }, + .cxc_count = 1, .pd = { .name = "video_subcore0", }, .parent = &video_top_gdsc.pd, .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL, }; static struct gdsc video_subcore1_gdsc = { .gdscr = 0x1044, + .cxcs = (unsigned int []){ 0x104c }, + .cxc_count = 1, .pd = { .name = "video_subcore1", }, .parent = &video_top_gdsc.pd, .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL, }; static struct gdsc mdss_gdsc = { diff --git a/drivers/clk/renesas/clk-r8a73a4.c b/drivers/clk/renesas/clk-r8a73a4.c index f45c2c45808b..4b1815147f77 100644 --- a/drivers/clk/renesas/clk-r8a73a4.c +++ b/drivers/clk/renesas/clk-r8a73a4.c @@ -30,8 +30,6 @@ struct r8a73a4_cpg { #define CPG_PLL2HCR 0xe4 #define CPG_PLL2SCR 0xf4 -#define CLK_ENABLE_ON_INIT BIT(0) - struct div4_clk { const char *name; unsigned int reg; diff --git a/drivers/clk/renesas/clk-r8a7740.c b/drivers/clk/renesas/clk-r8a7740.c index 3ee3f57e4e9a..22e9be7240bb 100644 --- a/drivers/clk/renesas/clk-r8a7740.c +++ b/drivers/clk/renesas/clk-r8a7740.c @@ -26,28 +26,25 @@ struct r8a7740_cpg { #define CPG_USBCKCR 0x8c #define CPG_FRQCRC 0xe0 -#define CLK_ENABLE_ON_INIT BIT(0) - struct div4_clk { const char *name; unsigned int reg; unsigned int shift; - int flags; }; static struct div4_clk div4_clks[] = { - { "i", CPG_FRQCRA, 20, CLK_ENABLE_ON_INIT }, - { "zg", CPG_FRQCRA, 16, CLK_ENABLE_ON_INIT }, - { "b", CPG_FRQCRA, 8, CLK_ENABLE_ON_INIT }, - { "m1", CPG_FRQCRA, 4, CLK_ENABLE_ON_INIT }, - { "hp", CPG_FRQCRB, 4, 0 }, - { "hpp", CPG_FRQCRC, 20, 0 }, - { "usbp", CPG_FRQCRC, 16, 0 }, - { "s", CPG_FRQCRC, 12, 0 }, - { "zb", CPG_FRQCRC, 8, 0 }, - { "m3", CPG_FRQCRC, 4, 0 }, - { "cp", CPG_FRQCRC, 0, 0 }, - { NULL, 0, 0, 0 }, + { "i", CPG_FRQCRA, 20 }, + { "zg", CPG_FRQCRA, 16 }, + { "b", CPG_FRQCRA, 8 }, + { "m1", CPG_FRQCRA, 4 }, + { "hp", CPG_FRQCRB, 4 }, + { "hpp", CPG_FRQCRC, 20 }, + { "usbp", CPG_FRQCRC, 16 }, + { "s", CPG_FRQCRC, 12 }, + { "zb", CPG_FRQCRC, 8 }, + { "m3", CPG_FRQCRC, 4 }, + { "cp", CPG_FRQCRC, 0 }, + { NULL, 0, 0 }, }; static const struct clk_div_table div4_div_table[] = { diff --git a/drivers/clk/renesas/clk-sh73a0.c b/drivers/clk/renesas/clk-sh73a0.c index 8c51090f13e1..47fc99ccd283 100644 --- a/drivers/clk/renesas/clk-sh73a0.c +++ b/drivers/clk/renesas/clk-sh73a0.c @@ -34,8 +34,6 @@ struct sh73a0_cpg { #define CPG_DSI0PHYCR 0x6c #define CPG_DSI1PHYCR 0x70 -#define CLK_ENABLE_ON_INIT BIT(0) - struct div4_clk { const char *name; const char *parent; diff --git a/drivers/clk/renesas/r8a779a0-cpg-mssr.c b/drivers/clk/renesas/r8a779a0-cpg-mssr.c index 4c2872f45387..ff3f85e906fe 100644 --- a/drivers/clk/renesas/r8a779a0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779a0-cpg-mssr.c @@ -139,7 +139,7 @@ static const struct mssr_mod_clk r8a779a0_mod_clks[] __initconst = { DEF_MOD("avb3", 214, R8A779A0_CLK_S3D2), DEF_MOD("avb4", 215, R8A779A0_CLK_S3D2), DEF_MOD("avb5", 216, R8A779A0_CLK_S3D2), - DEF_MOD("canfd0", 328, R8A779A0_CLK_CANFD), + DEF_MOD("canfd0", 328, R8A779A0_CLK_S3D2), DEF_MOD("csi40", 331, R8A779A0_CLK_CSI0), DEF_MOD("csi41", 400, R8A779A0_CLK_CSI0), DEF_MOD("csi42", 401, R8A779A0_CLK_CSI0), diff --git a/drivers/clk/renesas/r8a779h0-cpg-mssr.c b/drivers/clk/renesas/r8a779h0-cpg-mssr.c index 71f67a1c86d8..079b55b30b23 100644 --- a/drivers/clk/renesas/r8a779h0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779h0-cpg-mssr.c @@ -184,14 +184,35 @@ static const struct mssr_mod_clk r8a779h0_mod_clks[] = { DEF_MOD("i2c1", 519, R8A779H0_CLK_S0D6_PER), DEF_MOD("i2c2", 520, R8A779H0_CLK_S0D6_PER), DEF_MOD("i2c3", 521, R8A779H0_CLK_S0D6_PER), + DEF_MOD("irqc", 611, R8A779H0_CLK_CL16M), + DEF_MOD("msi0", 618, R8A779H0_CLK_MSO), + DEF_MOD("msi1", 619, R8A779H0_CLK_MSO), + DEF_MOD("msi2", 620, R8A779H0_CLK_MSO), + DEF_MOD("msi3", 621, R8A779H0_CLK_MSO), + DEF_MOD("msi4", 622, R8A779H0_CLK_MSO), + DEF_MOD("msi5", 623, R8A779H0_CLK_MSO), DEF_MOD("rpc-if", 629, R8A779H0_CLK_RPCD2), + DEF_MOD("scif0", 702, R8A779H0_CLK_SASYNCPERD4), + DEF_MOD("scif1", 703, R8A779H0_CLK_SASYNCPERD4), + DEF_MOD("scif3", 704, R8A779H0_CLK_SASYNCPERD4), + DEF_MOD("scif4", 705, R8A779H0_CLK_SASYNCPERD4), DEF_MOD("sdhi0", 706, R8A779H0_CLK_SD0), DEF_MOD("sydm1", 709, R8A779H0_CLK_S0D6_PER), DEF_MOD("sydm2", 710, R8A779H0_CLK_S0D6_PER), + DEF_MOD("tmu0", 713, R8A779H0_CLK_SASYNCRT), + DEF_MOD("tmu1", 714, R8A779H0_CLK_SASYNCPERD2), + DEF_MOD("tmu2", 715, R8A779H0_CLK_SASYNCPERD2), + DEF_MOD("tmu3", 716, R8A779H0_CLK_SASYNCPERD2), + DEF_MOD("tmu4", 717, R8A779H0_CLK_SASYNCPERD2), DEF_MOD("wdt1:wdt0", 907, R8A779H0_CLK_R), + DEF_MOD("cmt0", 910, R8A779H0_CLK_R), + DEF_MOD("cmt1", 911, R8A779H0_CLK_R), + DEF_MOD("cmt2", 912, R8A779H0_CLK_R), + DEF_MOD("cmt3", 913, R8A779H0_CLK_R), DEF_MOD("pfc0", 915, R8A779H0_CLK_CP), DEF_MOD("pfc1", 916, R8A779H0_CLK_CP), DEF_MOD("pfc2", 917, R8A779H0_CLK_CP), + DEF_MOD("tsc2:tsc1", 919, R8A779H0_CLK_CL16M), }; /* diff --git a/drivers/clk/renesas/r9a07g043-cpg.c b/drivers/clk/renesas/r9a07g043-cpg.c index 33532673d25d..16acc95f3c62 100644 --- a/drivers/clk/renesas/r9a07g043-cpg.c +++ b/drivers/clk/renesas/r9a07g043-cpg.c @@ -149,7 +149,7 @@ static const struct cpg_core_clk r9a07g043_core_clks[] __initconst = { #endif }; -static struct rzg2l_mod_clk r9a07g043_mod_clks[] = { +static const struct rzg2l_mod_clk r9a07g043_mod_clks[] = { #ifdef CONFIG_ARM64 DEF_MOD("gic", R9A07G043_GIC600_GICCLK, R9A07G043_CLK_P1, 0x514, 0), @@ -280,9 +280,13 @@ static struct rzg2l_mod_clk r9a07g043_mod_clks[] = { 0x5a8, 1), DEF_MOD("tsu_pclk", R9A07G043_TSU_PCLK, R9A07G043_CLK_TSU, 0x5ac, 0), +#ifdef CONFIG_RISCV + DEF_MOD("nceplic_aclk", R9A07G043_NCEPLIC_ACLK, R9A07G043_CLK_P1, + 0x608, 0), +#endif }; -static struct rzg2l_reset r9a07g043_resets[] = { +static const struct rzg2l_reset r9a07g043_resets[] = { #ifdef CONFIG_ARM64 DEF_RST(R9A07G043_GIC600_GICRESET_N, 0x814, 0), DEF_RST(R9A07G043_GIC600_DBG_GICRESET_N, 0x814, 1), @@ -338,6 +342,10 @@ static struct rzg2l_reset r9a07g043_resets[] = { DEF_RST(R9A07G043_ADC_PRESETN, 0x8a8, 0), DEF_RST(R9A07G043_ADC_ADRST_N, 0x8a8, 1), DEF_RST(R9A07G043_TSU_PRESETN, 0x8ac, 0), +#ifdef CONFIG_RISCV + DEF_RST(R9A07G043_NCEPLIC_ARESETN, 0x908, 0), +#endif + }; static const unsigned int r9a07g043_crit_mod_clks[] __initconst = { @@ -347,6 +355,7 @@ static const unsigned int r9a07g043_crit_mod_clks[] __initconst = { #endif #ifdef CONFIG_RISCV MOD_CLK_BASE + R9A07G043_IAX45_CLK, + MOD_CLK_BASE + R9A07G043_NCEPLIC_ACLK, #endif MOD_CLK_BASE + R9A07G043_DMAC_ACLK, }; diff --git a/drivers/clk/renesas/r9a07g044-cpg.c b/drivers/clk/renesas/r9a07g044-cpg.c index 48404cafea3f..f6df3f7a31b5 100644 --- a/drivers/clk/renesas/r9a07g044-cpg.c +++ b/drivers/clk/renesas/r9a07g044-cpg.c @@ -368,7 +368,7 @@ static const struct { #endif }; -static struct rzg2l_reset r9a07g044_resets[] = { +static const struct rzg2l_reset r9a07g044_resets[] = { DEF_RST(R9A07G044_GIC600_GICRESET_N, 0x814, 0), DEF_RST(R9A07G044_GIC600_DBG_GICRESET_N, 0x814, 1), DEF_RST(R9A07G044_IA55_RESETN, 0x818, 0), diff --git a/drivers/clk/renesas/r9a08g045-cpg.c b/drivers/clk/renesas/r9a08g045-cpg.c index c3e6da2de197..b068733b145f 100644 --- a/drivers/clk/renesas/r9a08g045-cpg.c +++ b/drivers/clk/renesas/r9a08g045-cpg.c @@ -240,6 +240,43 @@ static const unsigned int r9a08g045_crit_mod_clks[] __initconst = { MOD_CLK_BASE + R9A08G045_DMAC_ACLK, }; +static const struct rzg2l_cpg_pm_domain_init_data r9a08g045_pm_domains[] = { + /* Keep always-on domain on the first position for proper domains registration. */ + DEF_PD("always-on", R9A08G045_PD_ALWAYS_ON, + DEF_REG_CONF(0, 0), + RZG2L_PD_F_ALWAYS_ON), + DEF_PD("gic", R9A08G045_PD_GIC, + DEF_REG_CONF(CPG_BUS_ACPU_MSTOP, BIT(3)), + RZG2L_PD_F_ALWAYS_ON), + DEF_PD("ia55", R9A08G045_PD_IA55, + DEF_REG_CONF(CPG_BUS_PERI_CPU_MSTOP, BIT(13)), + RZG2L_PD_F_ALWAYS_ON), + DEF_PD("dmac", R9A08G045_PD_DMAC, + DEF_REG_CONF(CPG_BUS_REG1_MSTOP, GENMASK(3, 0)), + RZG2L_PD_F_ALWAYS_ON), + DEF_PD("wdt0", R9A08G045_PD_WDT0, + DEF_REG_CONF(CPG_BUS_REG0_MSTOP, BIT(0)), + RZG2L_PD_F_NONE), + DEF_PD("sdhi0", R9A08G045_PD_SDHI0, + DEF_REG_CONF(CPG_BUS_PERI_COM_MSTOP, BIT(0)), + RZG2L_PD_F_NONE), + DEF_PD("sdhi1", R9A08G045_PD_SDHI1, + DEF_REG_CONF(CPG_BUS_PERI_COM_MSTOP, BIT(1)), + RZG2L_PD_F_NONE), + DEF_PD("sdhi2", R9A08G045_PD_SDHI2, + DEF_REG_CONF(CPG_BUS_PERI_COM_MSTOP, BIT(11)), + RZG2L_PD_F_NONE), + DEF_PD("eth0", R9A08G045_PD_ETHER0, + DEF_REG_CONF(CPG_BUS_PERI_COM_MSTOP, BIT(2)), + RZG2L_PD_F_NONE), + DEF_PD("eth1", R9A08G045_PD_ETHER1, + DEF_REG_CONF(CPG_BUS_PERI_COM_MSTOP, BIT(3)), + RZG2L_PD_F_NONE), + DEF_PD("scif0", R9A08G045_PD_SCIF0, + DEF_REG_CONF(CPG_BUS_MCPU2_MSTOP, BIT(1)), + RZG2L_PD_F_NONE), +}; + const struct rzg2l_cpg_info r9a08g045_cpg_info = { /* Core Clocks */ .core_clks = r9a08g045_core_clks, @@ -260,5 +297,9 @@ const struct rzg2l_cpg_info r9a08g045_cpg_info = { .resets = r9a08g045_resets, .num_resets = R9A08G045_VBAT_BRESETN + 1, /* Last reset ID + 1 */ + /* Power domains */ + .pm_domains = r9a08g045_pm_domains, + .num_pm_domains = ARRAY_SIZE(r9a08g045_pm_domains), + .has_clk_mon_regs = true, }; diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c index 3d2daa4ba2a4..04b78064d4e0 100644 --- a/drivers/clk/renesas/rzg2l-cpg.c +++ b/drivers/clk/renesas/rzg2l-cpg.c @@ -139,7 +139,6 @@ struct rzg2l_pll5_mux_dsi_div_param { * @num_resets: Number of Module Resets in info->resets[] * @last_dt_core_clk: ID of the last Core Clock exported to DT * @info: Pointer to platform data - * @genpd: PM domain * @mux_dsi_div_params: pll5 mux and dsi div parameters */ struct rzg2l_cpg_priv { @@ -156,8 +155,6 @@ struct rzg2l_cpg_priv { const struct rzg2l_cpg_info *info; - struct generic_pm_domain genpd; - struct rzg2l_pll5_mux_dsi_div_param mux_dsi_div_params; }; @@ -1559,9 +1556,34 @@ static bool rzg2l_cpg_is_pm_clk(struct rzg2l_cpg_priv *priv, return true; } +/** + * struct rzg2l_cpg_pm_domains - RZ/G2L PM domains data structure + * @onecell_data: cell data + * @domains: generic PM domains + */ +struct rzg2l_cpg_pm_domains { + struct genpd_onecell_data onecell_data; + struct generic_pm_domain *domains[]; +}; + +/** + * struct rzg2l_cpg_pd - RZ/G2L power domain data structure + * @genpd: generic PM domain + * @priv: pointer to CPG private data structure + * @conf: CPG PM domain configuration info + * @id: RZ/G2L power domain ID + */ +struct rzg2l_cpg_pd { + struct generic_pm_domain genpd; + struct rzg2l_cpg_priv *priv; + struct rzg2l_cpg_pm_domain_conf conf; + u16 id; +}; + static int rzg2l_cpg_attach_dev(struct generic_pm_domain *domain, struct device *dev) { - struct rzg2l_cpg_priv *priv = container_of(domain, struct rzg2l_cpg_priv, genpd); + struct rzg2l_cpg_pd *pd = container_of(domain, struct rzg2l_cpg_pd, genpd); + struct rzg2l_cpg_priv *priv = pd->priv; struct device_node *np = dev->of_node; struct of_phandle_args clkspec; bool once = true; @@ -1618,30 +1640,179 @@ static void rzg2l_cpg_detach_dev(struct generic_pm_domain *unused, struct device static void rzg2l_cpg_genpd_remove(void *data) { + struct genpd_onecell_data *celldata = data; + + for (unsigned int i = 0; i < celldata->num_domains; i++) + pm_genpd_remove(celldata->domains[i]); +} + +static void rzg2l_cpg_genpd_remove_simple(void *data) +{ pm_genpd_remove(data); } +static int rzg2l_cpg_power_on(struct generic_pm_domain *domain) +{ + struct rzg2l_cpg_pd *pd = container_of(domain, struct rzg2l_cpg_pd, genpd); + struct rzg2l_cpg_reg_conf mstop = pd->conf.mstop; + struct rzg2l_cpg_priv *priv = pd->priv; + + /* Set MSTOP. */ + if (mstop.mask) + writel(mstop.mask << 16, priv->base + mstop.off); + + return 0; +} + +static int rzg2l_cpg_power_off(struct generic_pm_domain *domain) +{ + struct rzg2l_cpg_pd *pd = container_of(domain, struct rzg2l_cpg_pd, genpd); + struct rzg2l_cpg_reg_conf mstop = pd->conf.mstop; + struct rzg2l_cpg_priv *priv = pd->priv; + + /* Set MSTOP. */ + if (mstop.mask) + writel(mstop.mask | (mstop.mask << 16), priv->base + mstop.off); + + return 0; +} + +static int __init rzg2l_cpg_pd_setup(struct rzg2l_cpg_pd *pd, bool always_on) +{ + struct dev_power_governor *governor; + + pd->genpd.flags |= GENPD_FLAG_PM_CLK | GENPD_FLAG_ACTIVE_WAKEUP; + pd->genpd.attach_dev = rzg2l_cpg_attach_dev; + pd->genpd.detach_dev = rzg2l_cpg_detach_dev; + if (always_on) { + pd->genpd.flags |= GENPD_FLAG_ALWAYS_ON; + governor = &pm_domain_always_on_gov; + } else { + pd->genpd.power_on = rzg2l_cpg_power_on; + pd->genpd.power_off = rzg2l_cpg_power_off; + governor = &simple_qos_governor; + } + + return pm_genpd_init(&pd->genpd, governor, !always_on); +} + static int __init rzg2l_cpg_add_clk_domain(struct rzg2l_cpg_priv *priv) { struct device *dev = priv->dev; struct device_node *np = dev->of_node; - struct generic_pm_domain *genpd = &priv->genpd; + struct rzg2l_cpg_pd *pd; + int ret; + + pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL); + if (!pd) + return -ENOMEM; + + pd->genpd.name = np->name; + pd->priv = priv; + ret = rzg2l_cpg_pd_setup(pd, true); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, rzg2l_cpg_genpd_remove_simple, &pd->genpd); + if (ret) + return ret; + + return of_genpd_add_provider_simple(np, &pd->genpd); +} + +static struct generic_pm_domain * +rzg2l_cpg_pm_domain_xlate(const struct of_phandle_args *spec, void *data) +{ + struct generic_pm_domain *domain = ERR_PTR(-ENOENT); + struct genpd_onecell_data *genpd = data; + + if (spec->args_count != 1) + return ERR_PTR(-EINVAL); + + for (unsigned int i = 0; i < genpd->num_domains; i++) { + struct rzg2l_cpg_pd *pd = container_of(genpd->domains[i], struct rzg2l_cpg_pd, + genpd); + + if (pd->id == spec->args[0]) { + domain = &pd->genpd; + break; + } + } + + return domain; +} + +static int __init rzg2l_cpg_add_pm_domains(struct rzg2l_cpg_priv *priv) +{ + const struct rzg2l_cpg_info *info = priv->info; + struct device *dev = priv->dev; + struct device_node *np = dev->of_node; + struct rzg2l_cpg_pm_domains *domains; + struct generic_pm_domain *parent; + u32 ncells; int ret; - genpd->name = np->name; - genpd->flags = GENPD_FLAG_PM_CLK | GENPD_FLAG_ALWAYS_ON | - GENPD_FLAG_ACTIVE_WAKEUP; - genpd->attach_dev = rzg2l_cpg_attach_dev; - genpd->detach_dev = rzg2l_cpg_detach_dev; - ret = pm_genpd_init(genpd, &pm_domain_always_on_gov, false); + ret = of_property_read_u32(np, "#power-domain-cells", &ncells); + if (ret) + return ret; + + /* For backward compatibility. */ + if (!ncells) + return rzg2l_cpg_add_clk_domain(priv); + + domains = devm_kzalloc(dev, struct_size(domains, domains, info->num_pm_domains), + GFP_KERNEL); + if (!domains) + return -ENOMEM; + + domains->onecell_data.domains = domains->domains; + domains->onecell_data.num_domains = info->num_pm_domains; + domains->onecell_data.xlate = rzg2l_cpg_pm_domain_xlate; + + ret = devm_add_action_or_reset(dev, rzg2l_cpg_genpd_remove, &domains->onecell_data); if (ret) return ret; - ret = devm_add_action_or_reset(dev, rzg2l_cpg_genpd_remove, genpd); + for (unsigned int i = 0; i < info->num_pm_domains; i++) { + bool always_on = !!(info->pm_domains[i].flags & RZG2L_PD_F_ALWAYS_ON); + struct rzg2l_cpg_pd *pd; + + pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL); + if (!pd) + return -ENOMEM; + + pd->genpd.name = info->pm_domains[i].name; + pd->conf = info->pm_domains[i].conf; + pd->id = info->pm_domains[i].id; + pd->priv = priv; + + ret = rzg2l_cpg_pd_setup(pd, always_on); + if (ret) + return ret; + + if (always_on) { + ret = rzg2l_cpg_power_on(&pd->genpd); + if (ret) + return ret; + } + + domains->domains[i] = &pd->genpd; + /* Parent should be on the very first entry of info->pm_domains[]. */ + if (!i) { + parent = &pd->genpd; + continue; + } + + ret = pm_genpd_add_subdomain(parent, &pd->genpd); + if (ret) + return ret; + } + + ret = of_genpd_add_provider_onecell(np, &domains->onecell_data); if (ret) return ret; - return of_genpd_add_provider_simple(np, genpd); + return 0; } static int __init rzg2l_cpg_probe(struct platform_device *pdev) @@ -1697,7 +1868,7 @@ static int __init rzg2l_cpg_probe(struct platform_device *pdev) if (error) return error; - error = rzg2l_cpg_add_clk_domain(priv); + error = rzg2l_cpg_add_pm_domains(priv); if (error) return error; diff --git a/drivers/clk/renesas/rzg2l-cpg.h b/drivers/clk/renesas/rzg2l-cpg.h index 6e38c8fc888c..ecfe7e7ea8a1 100644 --- a/drivers/clk/renesas/rzg2l-cpg.h +++ b/drivers/clk/renesas/rzg2l-cpg.h @@ -27,6 +27,18 @@ #define CPG_PL6_ETH_SSEL (0x418) #define CPG_PL5_SDIV (0x420) #define CPG_RST_MON (0x680) +#define CPG_BUS_ACPU_MSTOP (0xB60) +#define CPG_BUS_MCPU1_MSTOP (0xB64) +#define CPG_BUS_MCPU2_MSTOP (0xB68) +#define CPG_BUS_PERI_COM_MSTOP (0xB6C) +#define CPG_BUS_PERI_CPU_MSTOP (0xB70) +#define CPG_BUS_PERI_DDR_MSTOP (0xB74) +#define CPG_BUS_REG0_MSTOP (0xB7C) +#define CPG_BUS_REG1_MSTOP (0xB80) +#define CPG_BUS_TZCDDR_MSTOP (0xB84) +#define CPG_MHU_MSTOP (0xB88) +#define CPG_BUS_MCPU3_MSTOP (0xB90) +#define CPG_BUS_PERI_CPU2_MSTOP (0xB94) #define CPG_OTHERFUNC1_REG (0xBE8) #define CPG_SIPLL5_STBY_RESETB BIT(0) @@ -235,6 +247,55 @@ struct rzg2l_reset { DEF_RST_MON(_id, _off, _bit, -1) /** + * struct rzg2l_cpg_reg_conf - RZ/G2L register configuration data structure + * @off: register offset + * @mask: register mask + */ +struct rzg2l_cpg_reg_conf { + u16 off; + u16 mask; +}; + +#define DEF_REG_CONF(_off, _mask) ((struct rzg2l_cpg_reg_conf) { .off = (_off), .mask = (_mask) }) + +/** + * struct rzg2l_cpg_pm_domain_conf - PM domain configuration data structure + * @mstop: MSTOP register configuration + */ +struct rzg2l_cpg_pm_domain_conf { + struct rzg2l_cpg_reg_conf mstop; +}; + +/** + * struct rzg2l_cpg_pm_domain_init_data - PM domain init data + * @name: PM domain name + * @conf: PM domain configuration + * @flags: RZG2L PM domain flags (see RZG2L_PD_F_*) + * @id: PM domain ID (similar to the ones defined in + * include/dt-bindings/clock/<soc-id>-cpg.h) + */ +struct rzg2l_cpg_pm_domain_init_data { + const char * const name; + struct rzg2l_cpg_pm_domain_conf conf; + u32 flags; + u16 id; +}; + +#define DEF_PD(_name, _id, _mstop_conf, _flags) \ + { \ + .name = (_name), \ + .id = (_id), \ + .conf = { \ + .mstop = (_mstop_conf), \ + }, \ + .flags = (_flags), \ + } + +/* Power domain flags. */ +#define RZG2L_PD_F_ALWAYS_ON BIT(0) +#define RZG2L_PD_F_NONE (0) + +/** * struct rzg2l_cpg_info - SoC-specific CPG Description * * @core_clks: Array of Core Clock definitions @@ -252,6 +313,8 @@ struct rzg2l_reset { * @crit_mod_clks: Array with Module Clock IDs of critical clocks that * should not be disabled without a knowledgeable driver * @num_crit_mod_clks: Number of entries in crit_mod_clks[] + * @pm_domains: PM domains init data array + * @num_pm_domains: Number of PM domains * @has_clk_mon_regs: Flag indicating whether the SoC has CLK_MON registers */ struct rzg2l_cpg_info { @@ -278,6 +341,10 @@ struct rzg2l_cpg_info { const unsigned int *crit_mod_clks; unsigned int num_crit_mod_clks; + /* Power domain. */ + const struct rzg2l_cpg_pm_domain_init_data *pm_domains; + unsigned int num_pm_domains; + bool has_clk_mon_regs; }; diff --git a/drivers/clk/rockchip/clk-mmc-phase.c b/drivers/clk/rockchip/clk-mmc-phase.c index 975454a3dd72..91012078681b 100644 --- a/drivers/clk/rockchip/clk-mmc-phase.c +++ b/drivers/clk/rockchip/clk-mmc-phase.c @@ -14,7 +14,6 @@ struct rockchip_mmc_clock { struct clk_hw hw; void __iomem *reg; - int id; int shift; int cached_phase; struct notifier_block clk_rate_change_nb; diff --git a/drivers/clk/rockchip/clk-rk3568.c b/drivers/clk/rockchip/clk-rk3568.c index 8cb21d10beca..53d10b1c627b 100644 --- a/drivers/clk/rockchip/clk-rk3568.c +++ b/drivers/clk/rockchip/clk-rk3568.c @@ -64,6 +64,7 @@ static struct rockchip_pll_rate_table rk3568_pll_rates[] = { RK3036_PLL_RATE(912000000, 1, 76, 2, 1, 1, 0), RK3036_PLL_RATE(816000000, 1, 68, 2, 1, 1, 0), RK3036_PLL_RATE(800000000, 3, 200, 2, 1, 1, 0), + RK3036_PLL_RATE(724000000, 3, 181, 2, 1, 1, 0), RK3036_PLL_RATE(700000000, 3, 350, 4, 1, 1, 0), RK3036_PLL_RATE(696000000, 1, 116, 4, 1, 1, 0), RK3036_PLL_RATE(600000000, 1, 100, 4, 1, 1, 0), @@ -215,6 +216,7 @@ static const struct rockchip_cpuclk_reg_data rk3568_cpuclk_data = { PNAME(mux_pll_p) = { "xin24m" }; PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc_32k" }; +PNAME(mux_usb480m_phy_p) = { "clk_usbphy0_480m", "clk_usbphy1_480m"}; PNAME(mux_armclk_p) = { "apll", "gpll" }; PNAME(clk_i2s0_8ch_tx_p) = { "clk_i2s0_8ch_tx_src", "clk_i2s0_8ch_tx_frac", "i2s0_mclkin", "xin_osc0_half" }; PNAME(clk_i2s0_8ch_rx_p) = { "clk_i2s0_8ch_rx_src", "clk_i2s0_8ch_rx_frac", "i2s0_mclkin", "xin_osc0_half" }; @@ -485,6 +487,9 @@ static struct rockchip_clk_branch rk3568_clk_branches[] __initdata = { MUX(USB480M, "usb480m", mux_usb480m_p, CLK_SET_RATE_PARENT, RK3568_MODE_CON0, 14, 2, MFLAGS), + MUX(USB480M_PHY, "usb480m_phy", mux_usb480m_phy_p, CLK_SET_RATE_PARENT, + RK3568_MISC_CON2, 15, 1, MFLAGS), + /* PD_CORE */ COMPOSITE(0, "sclk_core_src", apll_gpll_npll_p, CLK_IGNORE_UNUSED, RK3568_CLKSEL_CON(2), 8, 2, MFLAGS, 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, diff --git a/drivers/clk/rockchip/rst-rk3588.c b/drivers/clk/rockchip/rst-rk3588.c index e855bb8d5413..c4ebc01f1c9c 100644 --- a/drivers/clk/rockchip/rst-rk3588.c +++ b/drivers/clk/rockchip/rst-rk3588.c @@ -577,6 +577,7 @@ static const int rk3588_register_offset[] = { /* SOFTRST_CON59 */ RK3588_CRU_RESET_OFFSET(SRST_A_HDCP1_BIU, 59, 6), + RK3588_CRU_RESET_OFFSET(SRST_A_HDMIRX_BIU, 59, 7), RK3588_CRU_RESET_OFFSET(SRST_A_VO1_BIU, 59, 8), RK3588_CRU_RESET_OFFSET(SRST_H_VOP1_BIU, 59, 9), RK3588_CRU_RESET_OFFSET(SRST_H_VOP1_S_BIU, 59, 10), diff --git a/drivers/clk/samsung/clk-exynos-arm64.c b/drivers/clk/samsung/clk-exynos-arm64.c index 6fb7194df7ab..bf7de21f329e 100644 --- a/drivers/clk/samsung/clk-exynos-arm64.c +++ b/drivers/clk/samsung/clk-exynos-arm64.c @@ -17,10 +17,17 @@ #include "clk-exynos-arm64.h" +/* PLL register bits */ +#define PLL_CON1_MANUAL BIT(1) + /* Gate register bits */ #define GATE_MANUAL BIT(20) #define GATE_ENABLE_HWACG BIT(28) +/* PLL_CONx_PLL register offsets range */ +#define PLL_CON_OFF_START 0x100 +#define PLL_CON_OFF_END 0x600 + /* Gate register offsets range */ #define GATE_OFF_START 0x2000 #define GATE_OFF_END 0x2fff @@ -38,17 +45,36 @@ struct exynos_arm64_cmu_data { struct samsung_clk_provider *ctx; }; +/* Check if the register offset is a GATE register */ +static bool is_gate_reg(unsigned long off) +{ + return off >= GATE_OFF_START && off <= GATE_OFF_END; +} + +/* Check if the register offset is a PLL_CONx register */ +static bool is_pll_conx_reg(unsigned long off) +{ + return off >= PLL_CON_OFF_START && off <= PLL_CON_OFF_END; +} + +/* Check if the register offset is a PLL_CON1 register */ +static bool is_pll_con1_reg(unsigned long off) +{ + return is_pll_conx_reg(off) && (off & 0xf) == 0x4 && !(off & 0x10); +} + /** * exynos_arm64_init_clocks - Set clocks initial configuration - * @np: CMU device tree node with "reg" property (CMU addr) - * @reg_offs: Register offsets array for clocks to init - * @reg_offs_len: Number of register offsets in reg_offs array + * @np: CMU device tree node with "reg" property (CMU addr) + * @cmu: CMU data * - * Set manual control mode for all gate clocks. + * Set manual control mode for all gate and PLL clocks. */ static void __init exynos_arm64_init_clocks(struct device_node *np, - const unsigned long *reg_offs, size_t reg_offs_len) + const struct samsung_cmu_info *cmu) { + const unsigned long *reg_offs = cmu->clk_regs; + size_t reg_offs_len = cmu->nr_clk_regs; void __iomem *reg_base; size_t i; @@ -60,14 +86,14 @@ static void __init exynos_arm64_init_clocks(struct device_node *np, void __iomem *reg = reg_base + reg_offs[i]; u32 val; - /* Modify only gate clock registers */ - if (reg_offs[i] < GATE_OFF_START || reg_offs[i] > GATE_OFF_END) - continue; - - val = readl(reg); - val |= GATE_MANUAL; - val &= ~GATE_ENABLE_HWACG; - writel(val, reg); + if (cmu->manual_plls && is_pll_con1_reg(reg_offs[i])) { + writel(PLL_CON1_MANUAL, reg); + } else if (is_gate_reg(reg_offs[i])) { + val = readl(reg); + val |= GATE_MANUAL; + val &= ~GATE_ENABLE_HWACG; + writel(val, reg); + } } iounmap(reg_base); @@ -177,7 +203,7 @@ void __init exynos_arm64_register_cmu(struct device *dev, pr_err("%s: could not enable bus clock %s; err = %d\n", __func__, cmu->clk_name, err); - exynos_arm64_init_clocks(np, cmu->clk_regs, cmu->nr_clk_regs); + exynos_arm64_init_clocks(np, cmu); samsung_cmu_register_one(np, cmu); } @@ -224,7 +250,7 @@ int __init exynos_arm64_register_cmu_pm(struct platform_device *pdev, __func__, cmu->clk_name, ret); if (set_manual) - exynos_arm64_init_clocks(np, cmu->clk_regs, cmu->nr_clk_regs); + exynos_arm64_init_clocks(np, cmu); reg_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(reg_base)) diff --git a/drivers/clk/samsung/clk-exynos850.c b/drivers/clk/samsung/clk-exynos850.c index 82cfa22c0788..6215471c4ac6 100644 --- a/drivers/clk/samsung/clk-exynos850.c +++ b/drivers/clk/samsung/clk-exynos850.c @@ -14,13 +14,16 @@ #include <dt-bindings/clock/exynos850.h> #include "clk.h" +#include "clk-cpu.h" #include "clk-exynos-arm64.h" /* NOTE: Must be equal to the last clock ID increased by one */ -#define CLKS_NR_TOP (CLK_DOUT_G3D_SWITCH + 1) +#define CLKS_NR_TOP (CLK_DOUT_CPUCL1_SWITCH + 1) #define CLKS_NR_APM (CLK_GOUT_SYSREG_APM_PCLK + 1) #define CLKS_NR_AUD (CLK_GOUT_AUD_CMU_AUD_PCLK + 1) #define CLKS_NR_CMGP (CLK_GOUT_SYSREG_CMGP_PCLK + 1) +#define CLKS_NR_CPUCL0 (CLK_CLUSTER0_SCLK + 1) +#define CLKS_NR_CPUCL1 (CLK_CLUSTER1_SCLK + 1) #define CLKS_NR_G3D (CLK_GOUT_G3D_SYSREG_PCLK + 1) #define CLKS_NR_HSI (CLK_GOUT_HSI_CMU_HSI_PCLK + 1) #define CLKS_NR_IS (CLK_GOUT_IS_SYSREG_PCLK + 1) @@ -47,6 +50,10 @@ #define CLK_CON_MUX_MUX_CLKCMU_CORE_CCI 0x1018 #define CLK_CON_MUX_MUX_CLKCMU_CORE_MMC_EMBD 0x101c #define CLK_CON_MUX_MUX_CLKCMU_CORE_SSS 0x1020 +#define CLK_CON_MUX_MUX_CLKCMU_CPUCL0_DBG 0x1024 +#define CLK_CON_MUX_MUX_CLKCMU_CPUCL0_SWITCH 0x1028 +#define CLK_CON_MUX_MUX_CLKCMU_CPUCL1_DBG 0x102c +#define CLK_CON_MUX_MUX_CLKCMU_CPUCL1_SWITCH 0x1030 #define CLK_CON_MUX_MUX_CLKCMU_DPU 0x1034 #define CLK_CON_MUX_MUX_CLKCMU_G3D_SWITCH 0x1038 #define CLK_CON_MUX_MUX_CLKCMU_HSI_BUS 0x103c @@ -69,6 +76,10 @@ #define CLK_CON_DIV_CLKCMU_CORE_CCI 0x1824 #define CLK_CON_DIV_CLKCMU_CORE_MMC_EMBD 0x1828 #define CLK_CON_DIV_CLKCMU_CORE_SSS 0x182c +#define CLK_CON_DIV_CLKCMU_CPUCL0_DBG 0x1830 +#define CLK_CON_DIV_CLKCMU_CPUCL0_SWITCH 0x1834 +#define CLK_CON_DIV_CLKCMU_CPUCL1_DBG 0x1838 +#define CLK_CON_DIV_CLKCMU_CPUCL1_SWITCH 0x183c #define CLK_CON_DIV_CLKCMU_DPU 0x1840 #define CLK_CON_DIV_CLKCMU_G3D_SWITCH 0x1844 #define CLK_CON_DIV_CLKCMU_HSI_BUS 0x1848 @@ -97,6 +108,10 @@ #define CLK_CON_GAT_GATE_CLKCMU_CORE_CCI 0x2020 #define CLK_CON_GAT_GATE_CLKCMU_CORE_MMC_EMBD 0x2024 #define CLK_CON_GAT_GATE_CLKCMU_CORE_SSS 0x2028 +#define CLK_CON_GAT_GATE_CLKCMU_CPUCL0_DBG 0x202c +#define CLK_CON_GAT_GATE_CLKCMU_CPUCL0_SWITCH 0x2030 +#define CLK_CON_GAT_GATE_CLKCMU_CPUCL1_DBG 0x2034 +#define CLK_CON_GAT_GATE_CLKCMU_CPUCL1_SWITCH 0x2038 #define CLK_CON_GAT_GATE_CLKCMU_DPU 0x203c #define CLK_CON_GAT_GATE_CLKCMU_G3D_SWITCH 0x2040 #define CLK_CON_GAT_GATE_CLKCMU_HSI_BUS 0x2044 @@ -130,6 +145,10 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_MUX_MUX_CLKCMU_CORE_CCI, CLK_CON_MUX_MUX_CLKCMU_CORE_MMC_EMBD, CLK_CON_MUX_MUX_CLKCMU_CORE_SSS, + CLK_CON_MUX_MUX_CLKCMU_CPUCL0_DBG, + CLK_CON_MUX_MUX_CLKCMU_CPUCL0_SWITCH, + CLK_CON_MUX_MUX_CLKCMU_CPUCL1_DBG, + CLK_CON_MUX_MUX_CLKCMU_CPUCL1_SWITCH, CLK_CON_MUX_MUX_CLKCMU_DPU, CLK_CON_MUX_MUX_CLKCMU_G3D_SWITCH, CLK_CON_MUX_MUX_CLKCMU_HSI_BUS, @@ -152,6 +171,10 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_DIV_CLKCMU_CORE_CCI, CLK_CON_DIV_CLKCMU_CORE_MMC_EMBD, CLK_CON_DIV_CLKCMU_CORE_SSS, + CLK_CON_DIV_CLKCMU_CPUCL0_DBG, + CLK_CON_DIV_CLKCMU_CPUCL0_SWITCH, + CLK_CON_DIV_CLKCMU_CPUCL1_DBG, + CLK_CON_DIV_CLKCMU_CPUCL1_SWITCH, CLK_CON_DIV_CLKCMU_DPU, CLK_CON_DIV_CLKCMU_G3D_SWITCH, CLK_CON_DIV_CLKCMU_HSI_BUS, @@ -180,6 +203,10 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_GAT_GATE_CLKCMU_CORE_CCI, CLK_CON_GAT_GATE_CLKCMU_CORE_MMC_EMBD, CLK_CON_GAT_GATE_CLKCMU_CORE_SSS, + CLK_CON_GAT_GATE_CLKCMU_CPUCL0_DBG, + CLK_CON_GAT_GATE_CLKCMU_CPUCL0_SWITCH, + CLK_CON_GAT_GATE_CLKCMU_CPUCL1_DBG, + CLK_CON_GAT_GATE_CLKCMU_CPUCL1_SWITCH, CLK_CON_GAT_GATE_CLKCMU_DPU, CLK_CON_GAT_GATE_CLKCMU_G3D_SWITCH, CLK_CON_GAT_GATE_CLKCMU_HSI_BUS, @@ -234,6 +261,14 @@ PNAME(mout_core_mmc_embd_p) = { "oscclk", "dout_shared0_div2", "oscclk", "oscclk" }; PNAME(mout_core_sss_p) = { "dout_shared0_div3", "dout_shared1_div3", "dout_shared0_div4", "dout_shared1_div4" }; +/* List of parent clocks for Muxes in CMU_TOP: for CMU_CPUCL0 */ +PNAME(mout_cpucl0_switch_p) = { "fout_shared0_pll", "fout_shared1_pll", + "dout_shared0_div2", "dout_shared1_div2" }; +PNAME(mout_cpucl0_dbg_p) = { "dout_shared0_div4", "dout_shared1_div4" }; +/* List of parent clocks for Muxes in CMU_TOP: for CMU_CPUCL1 */ +PNAME(mout_cpucl1_switch_p) = { "fout_shared0_pll", "fout_shared1_pll", + "dout_shared0_div2", "dout_shared1_div2" }; +PNAME(mout_cpucl1_dbg_p) = { "dout_shared0_div4", "dout_shared1_div4" }; /* List of parent clocks for Muxes in CMU_TOP: for CMU_G3D */ PNAME(mout_g3d_switch_p) = { "dout_shared0_div2", "dout_shared1_div2", "dout_shared0_div3", "dout_shared1_div3" }; @@ -300,6 +335,18 @@ static const struct samsung_mux_clock top_mux_clks[] __initconst = { MUX(CLK_MOUT_CORE_SSS, "mout_core_sss", mout_core_sss_p, CLK_CON_MUX_MUX_CLKCMU_CORE_SSS, 0, 2), + /* CPUCL0 */ + MUX(CLK_MOUT_CPUCL0_DBG, "mout_cpucl0_dbg", mout_cpucl0_dbg_p, + CLK_CON_MUX_MUX_CLKCMU_CPUCL0_DBG, 0, 1), + MUX(CLK_MOUT_CPUCL0_SWITCH, "mout_cpucl0_switch", mout_cpucl0_switch_p, + CLK_CON_MUX_MUX_CLKCMU_CPUCL0_SWITCH, 0, 2), + + /* CPUCL1 */ + MUX(CLK_MOUT_CPUCL1_DBG, "mout_cpucl1_dbg", mout_cpucl1_dbg_p, + CLK_CON_MUX_MUX_CLKCMU_CPUCL1_DBG, 0, 1), + MUX(CLK_MOUT_CPUCL1_SWITCH, "mout_cpucl1_switch", mout_cpucl1_switch_p, + CLK_CON_MUX_MUX_CLKCMU_CPUCL1_SWITCH, 0, 2), + /* DPU */ MUX(CLK_MOUT_DPU, "mout_dpu", mout_dpu_p, CLK_CON_MUX_MUX_CLKCMU_DPU, 0, 2), @@ -378,6 +425,18 @@ static const struct samsung_div_clock top_div_clks[] __initconst = { DIV(CLK_DOUT_CORE_SSS, "dout_core_sss", "gout_core_sss", CLK_CON_DIV_CLKCMU_CORE_SSS, 0, 4), + /* CPUCL0 */ + DIV(CLK_DOUT_CPUCL0_DBG, "dout_cpucl0_dbg", "gout_cpucl0_dbg", + CLK_CON_DIV_CLKCMU_CPUCL0_DBG, 0, 3), + DIV(CLK_DOUT_CPUCL0_SWITCH, "dout_cpucl0_switch", "gout_cpucl0_switch", + CLK_CON_DIV_CLKCMU_CPUCL0_SWITCH, 0, 3), + + /* CPUCL1 */ + DIV(CLK_DOUT_CPUCL1_DBG, "dout_cpucl1_dbg", "gout_cpucl1_dbg", + CLK_CON_DIV_CLKCMU_CPUCL1_DBG, 0, 3), + DIV(CLK_DOUT_CPUCL1_SWITCH, "dout_cpucl1_switch", "gout_cpucl1_switch", + CLK_CON_DIV_CLKCMU_CPUCL1_SWITCH, 0, 3), + /* DPU */ DIV(CLK_DOUT_DPU, "dout_dpu", "gout_dpu", CLK_CON_DIV_CLKCMU_DPU, 0, 4), @@ -442,6 +501,18 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = { GATE(CLK_GOUT_AUD, "gout_aud", "mout_aud", CLK_CON_GAT_GATE_CLKCMU_AUD, 21, 0, 0), + /* CPUCL0 */ + GATE(CLK_GOUT_CPUCL0_DBG, "gout_cpucl0_dbg", "mout_cpucl0_dbg", + CLK_CON_GAT_GATE_CLKCMU_CPUCL0_DBG, 21, 0, 0), + GATE(CLK_GOUT_CPUCL0_SWITCH, "gout_cpucl0_switch", "mout_cpucl0_switch", + CLK_CON_GAT_GATE_CLKCMU_CPUCL0_SWITCH, 21, 0, 0), + + /* CPUCL1 */ + GATE(CLK_GOUT_CPUCL1_DBG, "gout_cpucl1_dbg", "mout_cpucl1_dbg", + CLK_CON_GAT_GATE_CLKCMU_CPUCL1_DBG, 21, 0, 0), + GATE(CLK_GOUT_CPUCL1_SWITCH, "gout_cpucl1_switch", "mout_cpucl1_switch", + CLK_CON_GAT_GATE_CLKCMU_CPUCL1_SWITCH, 21, 0, 0), + /* DPU */ GATE(CLK_GOUT_DPU, "gout_dpu", "mout_dpu", CLK_CON_GAT_GATE_CLKCMU_DPU, 21, 0, 0), @@ -1030,6 +1101,373 @@ static const struct samsung_cmu_info cmgp_cmu_info __initconst = { .clk_name = "gout_clkcmu_cmgp_bus", }; +/* ---- CMU_CPUCL0 ---------------------------------------------------------- */ + +/* Register Offset definitions for CMU_CPUCL0 (0x10900000) */ +#define PLL_LOCKTIME_PLL_CPUCL0 0x0000 +#define PLL_CON0_PLL_CPUCL0 0x0100 +#define PLL_CON1_PLL_CPUCL0 0x0104 +#define PLL_CON3_PLL_CPUCL0 0x010c +#define PLL_CON0_MUX_CLKCMU_CPUCL0_DBG_USER 0x0600 +#define PLL_CON0_MUX_CLKCMU_CPUCL0_SWITCH_USER 0x0610 +#define CLK_CON_MUX_MUX_CLK_CPUCL0_PLL 0x100c +#define CLK_CON_DIV_DIV_CLK_CLUSTER0_ACLK 0x1800 +#define CLK_CON_DIV_DIV_CLK_CLUSTER0_ATCLK 0x1808 +#define CLK_CON_DIV_DIV_CLK_CLUSTER0_PCLKDBG 0x180c +#define CLK_CON_DIV_DIV_CLK_CLUSTER0_PERIPHCLK 0x1810 +#define CLK_CON_DIV_DIV_CLK_CPUCL0_CMUREF 0x1814 +#define CLK_CON_DIV_DIV_CLK_CPUCL0_CPU 0x1818 +#define CLK_CON_DIV_DIV_CLK_CPUCL0_PCLK 0x181c +#define CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_ATCLK 0x2000 +#define CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PCLK 0x2004 +#define CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PERIPHCLK 0x2008 +#define CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_SCLK 0x200c +#define CLK_CON_GAT_CLK_CPUCL0_CMU_CPUCL0_PCLK 0x2010 +#define CLK_CON_GAT_GATE_CLK_CPUCL0_CPU 0x2020 + +static const unsigned long cpucl0_clk_regs[] __initconst = { + PLL_LOCKTIME_PLL_CPUCL0, + PLL_CON0_PLL_CPUCL0, + PLL_CON1_PLL_CPUCL0, + PLL_CON3_PLL_CPUCL0, + PLL_CON0_MUX_CLKCMU_CPUCL0_DBG_USER, + PLL_CON0_MUX_CLKCMU_CPUCL0_SWITCH_USER, + CLK_CON_MUX_MUX_CLK_CPUCL0_PLL, + CLK_CON_DIV_DIV_CLK_CLUSTER0_ACLK, + CLK_CON_DIV_DIV_CLK_CLUSTER0_ATCLK, + CLK_CON_DIV_DIV_CLK_CLUSTER0_PCLKDBG, + CLK_CON_DIV_DIV_CLK_CLUSTER0_PERIPHCLK, + CLK_CON_DIV_DIV_CLK_CPUCL0_CMUREF, + CLK_CON_DIV_DIV_CLK_CPUCL0_CPU, + CLK_CON_DIV_DIV_CLK_CPUCL0_PCLK, + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_ATCLK, + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PCLK, + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PERIPHCLK, + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_SCLK, + CLK_CON_GAT_CLK_CPUCL0_CMU_CPUCL0_PCLK, + CLK_CON_GAT_GATE_CLK_CPUCL0_CPU, +}; + +/* List of parent clocks for Muxes in CMU_CPUCL0 */ +PNAME(mout_pll_cpucl0_p) = { "oscclk", "fout_cpucl0_pll" }; +PNAME(mout_cpucl0_switch_user_p) = { "oscclk", "dout_cpucl0_switch" }; +PNAME(mout_cpucl0_dbg_user_p) = { "oscclk", "dout_cpucl0_dbg" }; +PNAME(mout_cpucl0_pll_p) = { "mout_pll_cpucl0", + "mout_cpucl0_switch_user" }; + +static const struct samsung_pll_rate_table cpu_pll_rates[] __initconst = { + PLL_35XX_RATE(26 * MHZ, 2210000000U, 255, 3, 0), + PLL_35XX_RATE(26 * MHZ, 2106000000U, 243, 3, 0), + PLL_35XX_RATE(26 * MHZ, 2002000000U, 231, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1846000000U, 213, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1742000000U, 201, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1586000000U, 183, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1456000000U, 168, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1300000000U, 150, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1157000000U, 267, 3, 1), + PLL_35XX_RATE(26 * MHZ, 1053000000U, 243, 3, 1), + PLL_35XX_RATE(26 * MHZ, 949000000U, 219, 3, 1), + PLL_35XX_RATE(26 * MHZ, 806000000U, 186, 3, 1), + PLL_35XX_RATE(26 * MHZ, 650000000U, 150, 3, 1), + PLL_35XX_RATE(26 * MHZ, 546000000U, 252, 3, 2), + PLL_35XX_RATE(26 * MHZ, 442000000U, 204, 3, 2), + PLL_35XX_RATE(26 * MHZ, 351000000U, 162, 3, 2), + PLL_35XX_RATE(26 * MHZ, 247000000U, 114, 3, 2), + PLL_35XX_RATE(26 * MHZ, 182000000U, 168, 3, 3), + PLL_35XX_RATE(26 * MHZ, 130000000U, 120, 3, 3), +}; + +static const struct samsung_pll_clock cpucl0_pll_clks[] __initconst = { + PLL(pll_0822x, CLK_FOUT_CPUCL0_PLL, "fout_cpucl0_pll", "oscclk", + PLL_LOCKTIME_PLL_CPUCL0, PLL_CON3_PLL_CPUCL0, cpu_pll_rates), +}; + +static const struct samsung_mux_clock cpucl0_mux_clks[] __initconst = { + MUX_F(CLK_MOUT_PLL_CPUCL0, "mout_pll_cpucl0", mout_pll_cpucl0_p, + PLL_CON0_PLL_CPUCL0, 4, 1, + CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0), + MUX_F(CLK_MOUT_CPUCL0_SWITCH_USER, "mout_cpucl0_switch_user", + mout_cpucl0_switch_user_p, + PLL_CON0_MUX_CLKCMU_CPUCL0_SWITCH_USER, 4, 1, + CLK_SET_RATE_PARENT, 0), + MUX(CLK_MOUT_CPUCL0_DBG_USER, "mout_cpucl0_dbg_user", + mout_cpucl0_dbg_user_p, + PLL_CON0_MUX_CLKCMU_CPUCL0_DBG_USER, 4, 1), + MUX_F(CLK_MOUT_CPUCL0_PLL, "mout_cpucl0_pll", mout_cpucl0_pll_p, + CLK_CON_MUX_MUX_CLK_CPUCL0_PLL, 0, 1, CLK_SET_RATE_PARENT, 0), +}; + +static const struct samsung_div_clock cpucl0_div_clks[] __initconst = { + DIV_F(CLK_DOUT_CPUCL0_CPU, "dout_cpucl0_cpu", "mout_cpucl0_pll", + CLK_CON_DIV_DIV_CLK_CPUCL0_CPU, 0, 1, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CPUCL0_CMUREF, "dout_cpucl0_cmuref", "dout_cpucl0_cpu", + CLK_CON_DIV_DIV_CLK_CPUCL0_CMUREF, 0, 3, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CPUCL0_PCLK, "dout_cpucl0_pclk", "dout_cpucl0_cpu", + CLK_CON_DIV_DIV_CLK_CPUCL0_PCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + + /* EMBEDDED_CMU_CPUCL0 */ + DIV_F(CLK_DOUT_CLUSTER0_ACLK, "dout_cluster0_aclk", "gout_cluster0_cpu", + CLK_CON_DIV_DIV_CLK_CLUSTER0_ACLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER0_ATCLK, "dout_cluster0_atclk", + "gout_cluster0_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER0_ATCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER0_PCLKDBG, "dout_cluster0_pclkdbg", + "gout_cluster0_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER0_PCLKDBG, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER0_PERIPHCLK, "dout_cluster0_periphclk", + "gout_cluster0_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER0_PERIPHCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), +}; + +static const struct samsung_gate_clock cpucl0_gate_clks[] __initconst = { + GATE(CLK_GOUT_CPUCL0_CMU_CPUCL0_PCLK, "gout_cpucl0_cmu_cpucl0_pclk", + "dout_cpucl0_pclk", + CLK_CON_GAT_CLK_CPUCL0_CMU_CPUCL0_PCLK, 21, CLK_IGNORE_UNUSED, 0), + + /* EMBEDDED_CMU_CPUCL0 */ + GATE(CLK_GOUT_CLUSTER0_CPU, "gout_cluster0_cpu", "dout_cpucl0_cpu", + CLK_CON_GAT_GATE_CLK_CPUCL0_CPU, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER0_SCLK, "gout_cluster0_sclk", "gout_cluster0_cpu", + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_SCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER0_ATCLK, "gout_cluster0_atclk", + "dout_cluster0_atclk", + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_ATCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER0_PERIPHCLK, "gout_cluster0_periphclk", + "dout_cluster0_periphclk", + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PERIPHCLK, 21, + CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER0_PCLK, "gout_cluster0_pclk", + "dout_cluster0_pclkdbg", + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PCLK, 21, CLK_IGNORE_UNUSED, 0), +}; + +/* + * Each parameter is going to be written into the corresponding DIV register. So + * the actual divider value for each parameter will be 1/(param+1). All these + * parameters must be in the range of 0..15, as the divider range for all of + * these DIV clocks is 1..16. The default values for these dividers is + * (1, 3, 3, 1). + */ +#define E850_CPU_DIV0(aclk, atclk, pclkdbg, periphclk) \ + (((aclk) << 16) | ((atclk) << 12) | ((pclkdbg) << 8) | \ + ((periphclk) << 4)) + +static const struct exynos_cpuclk_cfg_data exynos850_cluster_clk_d[] __initconst += { + { 2210000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 2106000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 2002000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1846000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1742000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1586000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1456000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1300000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1157000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1053000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 949000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 806000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 650000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 546000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 442000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 351000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 247000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 182000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 130000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 0 } +}; + +static const struct samsung_cpu_clock cpucl0_cpu_clks[] __initconst = { + CPU_CLK(CLK_CLUSTER0_SCLK, "cluster0_clk", CLK_MOUT_PLL_CPUCL0, + CLK_MOUT_CPUCL0_SWITCH_USER, 0, 0x0, CPUCLK_LAYOUT_E850_CL0, + exynos850_cluster_clk_d), +}; + +static const struct samsung_cmu_info cpucl0_cmu_info __initconst = { + .pll_clks = cpucl0_pll_clks, + .nr_pll_clks = ARRAY_SIZE(cpucl0_pll_clks), + .mux_clks = cpucl0_mux_clks, + .nr_mux_clks = ARRAY_SIZE(cpucl0_mux_clks), + .div_clks = cpucl0_div_clks, + .nr_div_clks = ARRAY_SIZE(cpucl0_div_clks), + .gate_clks = cpucl0_gate_clks, + .nr_gate_clks = ARRAY_SIZE(cpucl0_gate_clks), + .cpu_clks = cpucl0_cpu_clks, + .nr_cpu_clks = ARRAY_SIZE(cpucl0_cpu_clks), + .nr_clk_ids = CLKS_NR_CPUCL0, + .clk_regs = cpucl0_clk_regs, + .nr_clk_regs = ARRAY_SIZE(cpucl0_clk_regs), + .clk_name = "dout_cpucl0_switch", + .manual_plls = true, +}; + +static void __init exynos850_cmu_cpucl0_init(struct device_node *np) +{ + exynos_arm64_register_cmu(NULL, np, &cpucl0_cmu_info); +} + +/* Register CMU_CPUCL0 early, as CPU clocks should be available ASAP */ +CLK_OF_DECLARE(exynos850_cmu_cpucl0, "samsung,exynos850-cmu-cpucl0", + exynos850_cmu_cpucl0_init); + +/* ---- CMU_CPUCL1 ---------------------------------------------------------- */ + +/* Register Offset definitions for CMU_CPUCL1 (0x10800000) */ +#define PLL_LOCKTIME_PLL_CPUCL1 0x0000 +#define PLL_CON0_PLL_CPUCL1 0x0100 +#define PLL_CON1_PLL_CPUCL1 0x0104 +#define PLL_CON3_PLL_CPUCL1 0x010c +#define PLL_CON0_MUX_CLKCMU_CPUCL1_DBG_USER 0x0600 +#define PLL_CON0_MUX_CLKCMU_CPUCL1_SWITCH_USER 0x0610 +#define CLK_CON_MUX_MUX_CLK_CPUCL1_PLL 0x1000 +#define CLK_CON_DIV_DIV_CLK_CLUSTER1_ACLK 0x1800 +#define CLK_CON_DIV_DIV_CLK_CLUSTER1_ATCLK 0x1808 +#define CLK_CON_DIV_DIV_CLK_CLUSTER1_PCLKDBG 0x180c +#define CLK_CON_DIV_DIV_CLK_CLUSTER1_PERIPHCLK 0x1810 +#define CLK_CON_DIV_DIV_CLK_CPUCL1_CMUREF 0x1814 +#define CLK_CON_DIV_DIV_CLK_CPUCL1_CPU 0x1818 +#define CLK_CON_DIV_DIV_CLK_CPUCL1_PCLK 0x181c +#define CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_ATCLK 0x2000 +#define CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PCLK 0x2004 +#define CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PERIPHCLK 0x2008 +#define CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_SCLK 0x200c +#define CLK_CON_GAT_CLK_CPUCL1_CMU_CPUCL1_PCLK 0x2010 +#define CLK_CON_GAT_GATE_CLK_CPUCL1_CPU 0x2020 + +static const unsigned long cpucl1_clk_regs[] __initconst = { + PLL_LOCKTIME_PLL_CPUCL1, + PLL_CON0_PLL_CPUCL1, + PLL_CON1_PLL_CPUCL1, + PLL_CON3_PLL_CPUCL1, + PLL_CON0_MUX_CLKCMU_CPUCL1_DBG_USER, + PLL_CON0_MUX_CLKCMU_CPUCL1_SWITCH_USER, + CLK_CON_MUX_MUX_CLK_CPUCL1_PLL, + CLK_CON_DIV_DIV_CLK_CLUSTER1_ACLK, + CLK_CON_DIV_DIV_CLK_CLUSTER1_ATCLK, + CLK_CON_DIV_DIV_CLK_CLUSTER1_PCLKDBG, + CLK_CON_DIV_DIV_CLK_CLUSTER1_PERIPHCLK, + CLK_CON_DIV_DIV_CLK_CPUCL1_CMUREF, + CLK_CON_DIV_DIV_CLK_CPUCL1_CPU, + CLK_CON_DIV_DIV_CLK_CPUCL1_PCLK, + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_ATCLK, + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PCLK, + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PERIPHCLK, + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_SCLK, + CLK_CON_GAT_CLK_CPUCL1_CMU_CPUCL1_PCLK, + CLK_CON_GAT_GATE_CLK_CPUCL1_CPU, +}; + +/* List of parent clocks for Muxes in CMU_CPUCL0 */ +PNAME(mout_pll_cpucl1_p) = { "oscclk", "fout_cpucl1_pll" }; +PNAME(mout_cpucl1_switch_user_p) = { "oscclk", "dout_cpucl1_switch" }; +PNAME(mout_cpucl1_dbg_user_p) = { "oscclk", "dout_cpucl1_dbg" }; +PNAME(mout_cpucl1_pll_p) = { "mout_pll_cpucl1", + "mout_cpucl1_switch_user" }; + +static const struct samsung_pll_clock cpucl1_pll_clks[] __initconst = { + PLL(pll_0822x, CLK_FOUT_CPUCL1_PLL, "fout_cpucl1_pll", "oscclk", + PLL_LOCKTIME_PLL_CPUCL1, PLL_CON3_PLL_CPUCL1, cpu_pll_rates), +}; + +static const struct samsung_mux_clock cpucl1_mux_clks[] __initconst = { + MUX_F(CLK_MOUT_PLL_CPUCL1, "mout_pll_cpucl1", mout_pll_cpucl1_p, + PLL_CON0_PLL_CPUCL1, 4, 1, + CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0), + MUX_F(CLK_MOUT_CPUCL1_SWITCH_USER, "mout_cpucl1_switch_user", + mout_cpucl1_switch_user_p, + PLL_CON0_MUX_CLKCMU_CPUCL1_SWITCH_USER, 4, 1, + CLK_SET_RATE_PARENT, 0), + MUX(CLK_MOUT_CPUCL1_DBG_USER, "mout_cpucl1_dbg_user", + mout_cpucl1_dbg_user_p, + PLL_CON0_MUX_CLKCMU_CPUCL1_DBG_USER, 4, 1), + MUX_F(CLK_MOUT_CPUCL1_PLL, "mout_cpucl1_pll", mout_cpucl1_pll_p, + CLK_CON_MUX_MUX_CLK_CPUCL1_PLL, 0, 1, CLK_SET_RATE_PARENT, 0), +}; + +static const struct samsung_div_clock cpucl1_div_clks[] __initconst = { + DIV_F(CLK_DOUT_CPUCL1_CPU, "dout_cpucl1_cpu", "mout_cpucl1_pll", + CLK_CON_DIV_DIV_CLK_CPUCL1_CPU, 0, 1, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CPUCL1_CMUREF, "dout_cpucl1_cmuref", "dout_cpucl1_cpu", + CLK_CON_DIV_DIV_CLK_CPUCL1_CMUREF, 0, 3, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CPUCL1_PCLK, "dout_cpucl1_pclk", "dout_cpucl1_cpu", + CLK_CON_DIV_DIV_CLK_CPUCL1_PCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + + /* EMBEDDED_CMU_CPUCL1 */ + DIV_F(CLK_DOUT_CLUSTER1_ACLK, "dout_cluster1_aclk", "gout_cluster1_cpu", + CLK_CON_DIV_DIV_CLK_CLUSTER1_ACLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER1_ATCLK, "dout_cluster1_atclk", + "gout_cluster1_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER1_ATCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER1_PCLKDBG, "dout_cluster1_pclkdbg", + "gout_cluster1_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER1_PCLKDBG, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER1_PERIPHCLK, "dout_cluster1_periphclk", + "gout_cluster1_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER1_PERIPHCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), +}; + +static const struct samsung_gate_clock cpucl1_gate_clks[] __initconst = { + GATE(CLK_GOUT_CPUCL1_CMU_CPUCL1_PCLK, "gout_cpucl1_cmu_cpucl1_pclk", + "dout_cpucl1_pclk", + CLK_CON_GAT_CLK_CPUCL1_CMU_CPUCL1_PCLK, 21, CLK_IGNORE_UNUSED, 0), + + /* EMBEDDED_CMU_CPUCL1 */ + GATE(CLK_GOUT_CLUSTER1_CPU, "gout_cluster1_cpu", "dout_cpucl1_cpu", + CLK_CON_GAT_GATE_CLK_CPUCL1_CPU, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER1_SCLK, "gout_cluster1_sclk", "gout_cluster1_cpu", + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_SCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER1_ATCLK, "gout_cluster1_atclk", + "dout_cluster1_atclk", + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_ATCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER1_PERIPHCLK, "gout_cluster1_periphclk", + "dout_cluster1_periphclk", + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PERIPHCLK, 21, + CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER1_PCLK, "gout_cluster1_pclk", + "dout_cluster1_pclkdbg", + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PCLK, 21, CLK_IGNORE_UNUSED, 0), +}; + +static const struct samsung_cpu_clock cpucl1_cpu_clks[] __initconst = { + CPU_CLK(CLK_CLUSTER1_SCLK, "cluster1_clk", CLK_MOUT_PLL_CPUCL1, + CLK_MOUT_CPUCL1_SWITCH_USER, 0, 0x0, CPUCLK_LAYOUT_E850_CL1, + exynos850_cluster_clk_d), +}; + +static const struct samsung_cmu_info cpucl1_cmu_info __initconst = { + .pll_clks = cpucl1_pll_clks, + .nr_pll_clks = ARRAY_SIZE(cpucl1_pll_clks), + .mux_clks = cpucl1_mux_clks, + .nr_mux_clks = ARRAY_SIZE(cpucl1_mux_clks), + .div_clks = cpucl1_div_clks, + .nr_div_clks = ARRAY_SIZE(cpucl1_div_clks), + .gate_clks = cpucl1_gate_clks, + .nr_gate_clks = ARRAY_SIZE(cpucl1_gate_clks), + .cpu_clks = cpucl1_cpu_clks, + .nr_cpu_clks = ARRAY_SIZE(cpucl1_cpu_clks), + .nr_clk_ids = CLKS_NR_CPUCL1, + .clk_regs = cpucl1_clk_regs, + .nr_clk_regs = ARRAY_SIZE(cpucl1_clk_regs), + .clk_name = "dout_cpucl1_switch", + .manual_plls = true, +}; + +static void __init exynos850_cmu_cpucl1_init(struct device_node *np) +{ + exynos_arm64_register_cmu(NULL, np, &cpucl1_cmu_info); +} + +/* Register CMU_CPUCL1 early, as CPU clocks should be available ASAP */ +CLK_OF_DECLARE(exynos850_cmu_cpucl1, "samsung,exynos850-cmu-cpucl1", + exynos850_cmu_cpucl1_init); + /* ---- CMU_G3D ------------------------------------------------------------- */ /* Register Offset definitions for CMU_G3D (0x11400000) */ diff --git a/drivers/clk/samsung/clk-exynosautov9.c b/drivers/clk/samsung/clk-exynosautov9.c index e9c06eb93e66..f04bacacab2c 100644 --- a/drivers/clk/samsung/clk-exynosautov9.c +++ b/drivers/clk/samsung/clk-exynosautov9.c @@ -352,13 +352,13 @@ static const struct samsung_pll_clock top_pll_clks[] __initconst = { /* CMU_TOP_PURECLKCOMP */ PLL(pll_0822x, FOUT_SHARED0_PLL, "fout_shared0_pll", "oscclk", PLL_LOCKTIME_PLL_SHARED0, PLL_CON3_PLL_SHARED0, NULL), - PLL(pll_0822x, FOUT_SHARED0_PLL, "fout_shared1_pll", "oscclk", + PLL(pll_0822x, FOUT_SHARED1_PLL, "fout_shared1_pll", "oscclk", PLL_LOCKTIME_PLL_SHARED1, PLL_CON3_PLL_SHARED1, NULL), - PLL(pll_0822x, FOUT_SHARED0_PLL, "fout_shared2_pll", "oscclk", + PLL(pll_0822x, FOUT_SHARED2_PLL, "fout_shared2_pll", "oscclk", PLL_LOCKTIME_PLL_SHARED2, PLL_CON3_PLL_SHARED2, NULL), - PLL(pll_0822x, FOUT_SHARED0_PLL, "fout_shared3_pll", "oscclk", + PLL(pll_0822x, FOUT_SHARED3_PLL, "fout_shared3_pll", "oscclk", PLL_LOCKTIME_PLL_SHARED3, PLL_CON3_PLL_SHARED3, NULL), - PLL(pll_0822x, FOUT_SHARED0_PLL, "fout_shared4_pll", "oscclk", + PLL(pll_0822x, FOUT_SHARED4_PLL, "fout_shared4_pll", "oscclk", PLL_LOCKTIME_PLL_SHARED4, PLL_CON3_PLL_SHARED4, NULL), }; diff --git a/drivers/clk/samsung/clk-gs101.c b/drivers/clk/samsung/clk-gs101.c index d065e343a85d..ba9570f7a5fa 100644 --- a/drivers/clk/samsung/clk-gs101.c +++ b/drivers/clk/samsung/clk-gs101.c @@ -15,10 +15,13 @@ #include "clk.h" #include "clk-exynos-arm64.h" +#include "clk-pll.h" /* NOTE: Must be equal to the last clock ID increased by one */ #define CLKS_NR_TOP (CLK_GOUT_CMU_TPU_UART + 1) #define CLKS_NR_APM (CLK_APM_PLL_DIV16_APM + 1) +#define CLKS_NR_HSI0 (CLK_GOUT_HSI0_XIU_P_HSI0_ACLK + 1) +#define CLKS_NR_HSI2 (CLK_GOUT_HSI2_XIU_P_HSI2_ACLK + 1) #define CLKS_NR_MISC (CLK_GOUT_MISC_XIU_D_MISC_ACLK + 1) #define CLKS_NR_PERIC0 (CLK_GOUT_PERIC0_SYSREG_PERIC0_PCLK + 1) #define CLKS_NR_PERIC1 (CLK_GOUT_PERIC1_SYSREG_PERIC1_PCLK + 1) @@ -1893,16 +1896,16 @@ static const struct samsung_gate_clock apm_gate_clks[] __initconst = { CLK_CON_GAT_GOUT_BLK_APM_UID_UASC_G_SWD_IPCLKPORT_PCLK, 21, 0, 0), GATE(CLK_GOUT_APM_UASC_P_APM_ACLK, "gout_apm_uasc_p_apm_aclk", "gout_apm_func", - CLK_CON_GAT_GOUT_BLK_APM_UID_UASC_P_APM_IPCLKPORT_ACLK, 21, 0, 0), + CLK_CON_GAT_GOUT_BLK_APM_UID_UASC_P_APM_IPCLKPORT_ACLK, 21, CLK_IS_CRITICAL, 0), GATE(CLK_GOUT_APM_UASC_P_APM_PCLK, "gout_apm_uasc_p_apm_pclk", "gout_apm_func", - CLK_CON_GAT_GOUT_BLK_APM_UID_UASC_P_APM_IPCLKPORT_PCLK, 21, 0, 0), + CLK_CON_GAT_GOUT_BLK_APM_UID_UASC_P_APM_IPCLKPORT_PCLK, 21, CLK_IS_CRITICAL, 0), GATE(CLK_GOUT_APM_WDT_APM_PCLK, "gout_apm_wdt_apm_pclk", "gout_apm_func", CLK_CON_GAT_GOUT_BLK_APM_UID_WDT_APM_IPCLKPORT_PCLK, 21, 0, 0), GATE(CLK_GOUT_APM_XIU_DP_APM_ACLK, "gout_apm_xiu_dp_apm_aclk", "gout_apm_func", - CLK_CON_GAT_GOUT_BLK_APM_UID_XIU_DP_APM_IPCLKPORT_ACLK, 21, 0, 0), + CLK_CON_GAT_GOUT_BLK_APM_UID_XIU_DP_APM_IPCLKPORT_ACLK, 21, CLK_IS_CRITICAL, 0), }; static const struct samsung_cmu_info apm_cmu_info __initconst = { @@ -1919,6 +1922,958 @@ static const struct samsung_cmu_info apm_cmu_info __initconst = { .nr_clk_regs = ARRAY_SIZE(apm_clk_regs), }; +/* ---- CMU_HSI0 ------------------------------------------------------------ */ + +/* Register Offset definitions for CMU_HSI0 (0x11000000) */ +#define PLL_LOCKTIME_PLL_USB 0x0004 +#define PLL_CON0_PLL_USB 0x0140 +#define PLL_CON1_PLL_USB 0x0144 +#define PLL_CON2_PLL_USB 0x0148 +#define PLL_CON3_PLL_USB 0x014c +#define PLL_CON4_PLL_USB 0x0150 +#define PLL_CON0_MUX_CLKCMU_HSI0_ALT_USER 0x0600 +#define PLL_CON1_MUX_CLKCMU_HSI0_ALT_USER 0x0604 +#define PLL_CON0_MUX_CLKCMU_HSI0_BUS_USER 0x0610 +#define PLL_CON1_MUX_CLKCMU_HSI0_BUS_USER 0x0614 +#define PLL_CON0_MUX_CLKCMU_HSI0_DPGTC_USER 0x0620 +#define PLL_CON1_MUX_CLKCMU_HSI0_DPGTC_USER 0x0624 +#define PLL_CON0_MUX_CLKCMU_HSI0_TCXO_USER 0x0630 +#define PLL_CON1_MUX_CLKCMU_HSI0_TCXO_USER 0x0634 +#define PLL_CON0_MUX_CLKCMU_HSI0_USB20_USER 0x0640 +#define PLL_CON1_MUX_CLKCMU_HSI0_USB20_USER 0x0644 +#define PLL_CON0_MUX_CLKCMU_HSI0_USB31DRD_USER 0x0650 +#define PLL_CON1_MUX_CLKCMU_HSI0_USB31DRD_USER 0x0654 +#define PLL_CON0_MUX_CLKCMU_HSI0_USPDPDBG_USER 0x0660 +#define PLL_CON1_MUX_CLKCMU_HSI0_USPDPDBG_USER 0x0664 +#define HSI0_CMU_HSI0_CONTROLLER_OPTION 0x0800 +#define CLKOUT_CON_BLK_HSI0_CMU_HSI0_CLKOUT0 0x0810 +#define CLK_CON_MUX_MUX_CLK_HSI0_BUS 0x1000 +#define CLK_CON_MUX_MUX_CLK_HSI0_USB20_REF 0x1004 +#define CLK_CON_MUX_MUX_CLK_HSI0_USB31DRD 0x1008 +#define CLK_CON_DIV_DIV_CLK_HSI0_USB31DRD 0x1800 +#define CLK_CON_GAT_CLK_BLK_HSI0_UID_HSI0_CMU_HSI0_IPCLKPORT_PCLK 0x2000 +#define CLK_CON_GAT_CLK_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_SUSPEND_CLK_26 0x2004 +#define CLK_CON_GAT_CLK_HSI0_ALT 0x2008 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_DP_GTC_CLK 0x200c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_PCLK 0x2010 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_D_TZPC_HSI0_IPCLKPORT_PCLK 0x2014 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_ACLK 0x2018 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_PCLK 0x201c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_GPC_HSI0_IPCLKPORT_PCLK 0x2020 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_G_ETR_HSI0_IPCLKPORT_I_CLK 0x2024 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_AOCHSI0_IPCLKPORT_I_CLK 0x2028 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_HSI0_IPCLKPORT_I_CLK 0x202c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_ACEL_D_HSI0_IPCLKPORT_I_CLK 0x2030 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_AXI_D_HSI0AOC_IPCLKPORT_I_CLK 0x2034 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_ACLK 0x2038 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_PCLK 0x203c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_ACLK 0x2040 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_PCLK 0x2044 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_RSTNSYNC_CLK_HSI0_BUS_IPCLKPORT_CLK 0x2048 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_ACLK 0x204c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_PCLK 0x2050 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSMMU_USB_IPCLKPORT_CLK_S2 0x2054 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSREG_HSI0_IPCLKPORT_PCLK 0x2058 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_ACLK 0x205c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_PCLK 0x2060 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_ACLK 0x2064 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_PCLK 0x2068 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_ACLK_PHYCTRL 0x206c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_BUS_CLK_EARLY 0x2070 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB20_PHY_REFCLK_26 0x2074 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_REF_CLK_40 0x2078 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_REF_SOC_PLL 0x207c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_SCL_APB_PCLK 0x2080 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBPCS_APB_CLK 0x2084 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_I_ACLK 0x2088 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_UDBG_I_APB_PCLK 0x208c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D0_HSI0_IPCLKPORT_ACLK 0x2090 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D1_HSI0_IPCLKPORT_ACLK 0x2094 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_P_HSI0_IPCLKPORT_ACLK 0x2098 +#define DMYQCH_CON_USB31DRD_QCH 0x3000 +#define DMYQCH_CON_USB31DRD_QCH_REF 0x3004 +#define PCH_CON_LHM_AXI_G_ETR_HSI0_PCH 0x3008 +#define PCH_CON_LHM_AXI_P_AOCHSI0_PCH 0x300c +#define PCH_CON_LHM_AXI_P_HSI0_PCH 0x3010 +#define PCH_CON_LHS_ACEL_D_HSI0_PCH 0x3014 +#define PCH_CON_LHS_AXI_D_HSI0AOC_PCH 0x3018 +#define QCH_CON_DP_LINK_QCH_GTC_CLK 0x301c +#define QCH_CON_DP_LINK_QCH_PCLK 0x3020 +#define QCH_CON_D_TZPC_HSI0_QCH 0x3024 +#define QCH_CON_ETR_MIU_QCH_ACLK 0x3028 +#define QCH_CON_ETR_MIU_QCH_PCLK 0x302c +#define QCH_CON_GPC_HSI0_QCH 0x3030 +#define QCH_CON_HSI0_CMU_HSI0_QCH 0x3034 +#define QCH_CON_LHM_AXI_G_ETR_HSI0_QCH 0x3038 +#define QCH_CON_LHM_AXI_P_AOCHSI0_QCH 0x303c +#define QCH_CON_LHM_AXI_P_HSI0_QCH 0x3040 +#define QCH_CON_LHS_ACEL_D_HSI0_QCH 0x3044 +#define QCH_CON_LHS_AXI_D_HSI0AOC_QCH 0x3048 +#define QCH_CON_PPMU_HSI0_AOC_QCH 0x304c +#define QCH_CON_PPMU_HSI0_BUS0_QCH 0x3050 +#define QCH_CON_SSMT_USB_QCH 0x3054 +#define QCH_CON_SYSMMU_USB_QCH 0x3058 +#define QCH_CON_SYSREG_HSI0_QCH 0x305c +#define QCH_CON_UASC_HSI0_CTRL_QCH 0x3060 +#define QCH_CON_UASC_HSI0_LINK_QCH 0x3064 +#define QCH_CON_USB31DRD_QCH_APB 0x3068 +#define QCH_CON_USB31DRD_QCH_DBG 0x306c +#define QCH_CON_USB31DRD_QCH_PCS 0x3070 +#define QCH_CON_USB31DRD_QCH_SLV_CTRL 0x3074 +#define QCH_CON_USB31DRD_QCH_SLV_LINK 0x3078 +#define QUEUE_CTRL_REG_BLK_HSI0_CMU_HSI0 0x3c00 + +static const unsigned long hsi0_clk_regs[] __initconst = { + PLL_LOCKTIME_PLL_USB, + PLL_CON0_PLL_USB, + PLL_CON1_PLL_USB, + PLL_CON2_PLL_USB, + PLL_CON3_PLL_USB, + PLL_CON4_PLL_USB, + PLL_CON0_MUX_CLKCMU_HSI0_ALT_USER, + PLL_CON1_MUX_CLKCMU_HSI0_ALT_USER, + PLL_CON0_MUX_CLKCMU_HSI0_BUS_USER, + PLL_CON1_MUX_CLKCMU_HSI0_BUS_USER, + PLL_CON0_MUX_CLKCMU_HSI0_DPGTC_USER, + PLL_CON1_MUX_CLKCMU_HSI0_DPGTC_USER, + PLL_CON0_MUX_CLKCMU_HSI0_TCXO_USER, + PLL_CON1_MUX_CLKCMU_HSI0_TCXO_USER, + PLL_CON0_MUX_CLKCMU_HSI0_USB20_USER, + PLL_CON1_MUX_CLKCMU_HSI0_USB20_USER, + PLL_CON0_MUX_CLKCMU_HSI0_USB31DRD_USER, + PLL_CON1_MUX_CLKCMU_HSI0_USB31DRD_USER, + PLL_CON0_MUX_CLKCMU_HSI0_USPDPDBG_USER, + PLL_CON1_MUX_CLKCMU_HSI0_USPDPDBG_USER, + HSI0_CMU_HSI0_CONTROLLER_OPTION, + CLKOUT_CON_BLK_HSI0_CMU_HSI0_CLKOUT0, + CLK_CON_MUX_MUX_CLK_HSI0_BUS, + CLK_CON_MUX_MUX_CLK_HSI0_USB20_REF, + CLK_CON_MUX_MUX_CLK_HSI0_USB31DRD, + CLK_CON_DIV_DIV_CLK_HSI0_USB31DRD, + CLK_CON_GAT_CLK_BLK_HSI0_UID_HSI0_CMU_HSI0_IPCLKPORT_PCLK, + CLK_CON_GAT_CLK_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_SUSPEND_CLK_26, + CLK_CON_GAT_CLK_HSI0_ALT, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_DP_GTC_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_D_TZPC_HSI0_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_GPC_HSI0_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_G_ETR_HSI0_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_AOCHSI0_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_HSI0_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_ACEL_D_HSI0_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_AXI_D_HSI0AOC_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_RSTNSYNC_CLK_HSI0_BUS_IPCLKPORT_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSMMU_USB_IPCLKPORT_CLK_S2, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSREG_HSI0_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_ACLK_PHYCTRL, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_BUS_CLK_EARLY, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB20_PHY_REFCLK_26, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_REF_CLK_40, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_REF_SOC_PLL, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_SCL_APB_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBPCS_APB_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_I_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_UDBG_I_APB_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D0_HSI0_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D1_HSI0_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_P_HSI0_IPCLKPORT_ACLK, + DMYQCH_CON_USB31DRD_QCH, + DMYQCH_CON_USB31DRD_QCH_REF, + PCH_CON_LHM_AXI_G_ETR_HSI0_PCH, + PCH_CON_LHM_AXI_P_AOCHSI0_PCH, + PCH_CON_LHM_AXI_P_HSI0_PCH, + PCH_CON_LHS_ACEL_D_HSI0_PCH, + PCH_CON_LHS_AXI_D_HSI0AOC_PCH, + QCH_CON_DP_LINK_QCH_GTC_CLK, + QCH_CON_DP_LINK_QCH_PCLK, + QCH_CON_D_TZPC_HSI0_QCH, + QCH_CON_ETR_MIU_QCH_ACLK, + QCH_CON_ETR_MIU_QCH_PCLK, + QCH_CON_GPC_HSI0_QCH, + QCH_CON_HSI0_CMU_HSI0_QCH, + QCH_CON_LHM_AXI_G_ETR_HSI0_QCH, + QCH_CON_LHM_AXI_P_AOCHSI0_QCH, + QCH_CON_LHM_AXI_P_HSI0_QCH, + QCH_CON_LHS_ACEL_D_HSI0_QCH, + QCH_CON_LHS_AXI_D_HSI0AOC_QCH, + QCH_CON_PPMU_HSI0_AOC_QCH, + QCH_CON_PPMU_HSI0_BUS0_QCH, + QCH_CON_SSMT_USB_QCH, + QCH_CON_SYSMMU_USB_QCH, + QCH_CON_SYSREG_HSI0_QCH, + QCH_CON_UASC_HSI0_CTRL_QCH, + QCH_CON_UASC_HSI0_LINK_QCH, + QCH_CON_USB31DRD_QCH_APB, + QCH_CON_USB31DRD_QCH_DBG, + QCH_CON_USB31DRD_QCH_PCS, + QCH_CON_USB31DRD_QCH_SLV_CTRL, + QCH_CON_USB31DRD_QCH_SLV_LINK, + QUEUE_CTRL_REG_BLK_HSI0_CMU_HSI0, +}; + +/* List of parent clocks for Muxes in CMU_HSI0 */ +PNAME(mout_pll_usb_p) = { "oscclk", "fout_usb_pll" }; +PNAME(mout_hsi0_alt_user_p) = { "oscclk", + "gout_hsi0_clk_hsi0_alt" }; +PNAME(mout_hsi0_bus_user_p) = { "oscclk", "dout_cmu_hsi0_bus" }; +PNAME(mout_hsi0_dpgtc_user_p) = { "oscclk", "dout_cmu_hsi0_dpgtc" }; +PNAME(mout_hsi0_tcxo_user_p) = { "oscclk", "tcxo_hsi1_hsi0" }; +PNAME(mout_hsi0_usb20_user_p) = { "oscclk", "usb20phy_phy_clock" }; +PNAME(mout_hsi0_usb31drd_user_p) = { "oscclk", + "dout_cmu_hsi0_usb31drd" }; +PNAME(mout_hsi0_usbdpdbg_user_p) = { "oscclk", + "dout_cmu_hsi0_usbdpdbg" }; +PNAME(mout_hsi0_bus_p) = { "mout_hsi0_bus_user", + "mout_hsi0_alt_user" }; +PNAME(mout_hsi0_usb20_ref_p) = { "fout_usb_pll", + "mout_hsi0_tcxo_user" }; +PNAME(mout_hsi0_usb31drd_p) = { "fout_usb_pll", + "mout_hsi0_usb31drd_user", + "dout_hsi0_usb31drd", + "fout_usb_pll" }; + +static const struct samsung_pll_rate_table cmu_hsi0_usb_pll_rates[] __initconst = { + PLL_35XX_RATE(24576000, 19200000, 150, 6, 5), + { /* sentinel */ } +}; + +static const struct samsung_pll_clock cmu_hsi0_pll_clks[] __initconst = { + PLL(pll_0518x, CLK_FOUT_USB_PLL, "fout_usb_pll", "oscclk", + PLL_LOCKTIME_PLL_USB, PLL_CON3_PLL_USB, + cmu_hsi0_usb_pll_rates), +}; + +static const struct samsung_mux_clock hsi0_mux_clks[] __initconst = { + MUX(CLK_MOUT_PLL_USB, + "mout_pll_usb", mout_pll_usb_p, + PLL_CON0_PLL_USB, 4, 1), + MUX(CLK_MOUT_HSI0_ALT_USER, + "mout_hsi0_alt_user", mout_hsi0_alt_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_ALT_USER, 4, 1), + MUX(CLK_MOUT_HSI0_BUS_USER, + "mout_hsi0_bus_user", mout_hsi0_bus_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_BUS_USER, 4, 1), + MUX(CLK_MOUT_HSI0_DPGTC_USER, + "mout_hsi0_dpgtc_user", mout_hsi0_dpgtc_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_DPGTC_USER, 4, 1), + MUX(CLK_MOUT_HSI0_TCXO_USER, + "mout_hsi0_tcxo_user", mout_hsi0_tcxo_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_TCXO_USER, 4, 1), + MUX(CLK_MOUT_HSI0_USB20_USER, + "mout_hsi0_usb20_user", mout_hsi0_usb20_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_USB20_USER, 4, 1), + MUX(CLK_MOUT_HSI0_USB31DRD_USER, + "mout_hsi0_usb31drd_user", mout_hsi0_usb31drd_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_USB31DRD_USER, 4, 1), + MUX(CLK_MOUT_HSI0_USBDPDBG_USER, + "mout_hsi0_usbdpdbg_user", mout_hsi0_usbdpdbg_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_USPDPDBG_USER, 4, 1), + MUX(CLK_MOUT_HSI0_BUS, + "mout_hsi0_bus", mout_hsi0_bus_p, + CLK_CON_MUX_MUX_CLK_HSI0_BUS, 0, 1), + MUX(CLK_MOUT_HSI0_USB20_REF, + "mout_hsi0_usb20_ref", mout_hsi0_usb20_ref_p, + CLK_CON_MUX_MUX_CLK_HSI0_USB20_REF, 0, 1), + MUX(CLK_MOUT_HSI0_USB31DRD, + "mout_hsi0_usb31drd", mout_hsi0_usb31drd_p, + CLK_CON_MUX_MUX_CLK_HSI0_USB31DRD, 0, 2), +}; + +static const struct samsung_div_clock hsi0_div_clks[] __initconst = { + DIV(CLK_DOUT_HSI0_USB31DRD, + "dout_hsi0_usb31drd", "mout_hsi0_usb20_user", + CLK_CON_DIV_DIV_CLK_HSI0_USB31DRD, 0, 3), +}; + +static const struct samsung_gate_clock hsi0_gate_clks[] __initconst = { + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_PCLK, + "gout_hsi0_hsi0_pclk", "mout_hsi0_bus", + CLK_CON_GAT_CLK_BLK_HSI0_UID_HSI0_CMU_HSI0_IPCLKPORT_PCLK, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USB31DRD_SUSPEND_CLK_26, + "gout_hsi0_usb31drd_i_usb31drd_suspend_clk_26", + "mout_hsi0_usb20_ref", + CLK_CON_GAT_CLK_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_SUSPEND_CLK_26, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_CLK_HSI0_ALT, + "gout_hsi0_clk_hsi0_alt", "ioclk_clk_hsi0_alt", + CLK_CON_GAT_CLK_HSI0_ALT, 21, 0, 0), + GATE(CLK_GOUT_HSI0_DP_LINK_I_DP_GTC_CLK, + "gout_hsi0_dp_link_i_dp_gtc_clk", "mout_hsi0_dpgtc_user", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_DP_GTC_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_DP_LINK_I_PCLK, + "gout_hsi0_dp_link_i_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI0_D_TZPC_HSI0_PCLK, + "gout_hsi0_d_tzpc_hsi0_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_D_TZPC_HSI0_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_ETR_MIU_I_ACLK, + "gout_hsi0_etr_miu_i_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_ACLK, 21, 0, 0), + GATE(CLK_GOUT_HSI0_ETR_MIU_I_PCLK, + "gout_hsi0_etr_miu_i_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI0_GPC_HSI0_PCLK, + "gout_hsi0_gpc_hsi0_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_GPC_HSI0_IPCLKPORT_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI0_LHM_AXI_G_ETR_HSI0_I_CLK, + "gout_hsi0_lhm_axi_g_etr_hsi0_i_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_G_ETR_HSI0_IPCLKPORT_I_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_LHM_AXI_P_AOCHSI0_I_CLK, + "gout_hsi0_lhm_axi_p_aochsi0_i_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_AOCHSI0_IPCLKPORT_I_CLK, + 21, 0, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_LHM_AXI_P_HSI0_I_CLK, + "gout_hsi0_lhm_axi_p_hsi0_i_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_HSI0_IPCLKPORT_I_CLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_LHS_ACEL_D_HSI0_I_CLK, + "gout_hsi0_lhs_acel_d_hsi0_i_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_ACEL_D_HSI0_IPCLKPORT_I_CLK, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI0_LHS_AXI_D_HSI0AOC_I_CLK, + "gout_hsi0_lhs_axi_d_hsi0aoc_i_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_AXI_D_HSI0AOC_IPCLKPORT_I_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_PPMU_HSI0_AOC_ACLK, + "gout_hsi0_ppmu_hsi0_aoc_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_PPMU_HSI0_AOC_PCLK, + "gout_hsi0_ppmu_hsi0_aoc_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_PPMU_HSI0_BUS0_ACLK, + "gout_hsi0_ppmu_hsi0_bus0_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_PPMU_HSI0_BUS0_PCLK, + "gout_hsi0_ppmu_hsi0_bus0_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_CLK_HSI0_BUS_CLK, + "gout_hsi0_clk_hsi0_bus_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_RSTNSYNC_CLK_HSI0_BUS_IPCLKPORT_CLK, + 21, 0, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_SSMT_USB_ACLK, + "gout_hsi0_ssmt_usb_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_SSMT_USB_PCLK, + "gout_hsi0_ssmt_usb_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_PCLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_SYSMMU_USB_CLK_S2, + "gout_hsi0_sysmmu_usb_clk_s2", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSMMU_USB_IPCLKPORT_CLK_S2, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI0_SYSREG_HSI0_PCLK, + "gout_hsi0_sysreg_hsi0_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSREG_HSI0_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_UASC_HSI0_CTRL_ACLK, + "gout_hsi0_uasc_hsi0_ctrl_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_UASC_HSI0_CTRL_PCLK, + "gout_hsi0_uasc_hsi0_ctrl_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_UASC_HSI0_LINK_ACLK, + "gout_hsi0_uasc_hsi0_link_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_UASC_HSI0_LINK_PCLK, + "gout_hsi0_uasc_hsi0_link_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_ACLK_PHYCTRL, + "gout_hsi0_usb31drd_aclk_phyctrl", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_ACLK_PHYCTRL, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_BUS_CLK_EARLY, + "gout_hsi0_usb31drd_bus_clk_early", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_BUS_CLK_EARLY, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USB20_PHY_REFCLK_26, + "gout_hsi0_usb31drd_i_usb20_phy_refclk_26", "mout_hsi0_usb20_ref", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB20_PHY_REFCLK_26, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USB31DRD_REF_CLK_40, + "gout_hsi0_usb31drd_i_usb31drd_ref_clk_40", "mout_hsi0_usb31drd", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_REF_CLK_40, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USBDPPHY_REF_SOC_PLL, + "gout_hsi0_usb31drd_i_usbdpphy_ref_soc_pll", + "mout_hsi0_usbdpdbg_user", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_REF_SOC_PLL, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USBDPPHY_SCL_APB_PCLK, + "gout_hsi0_usb31drd_i_usbdpphy_scl_apb_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_SCL_APB_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USBPCS_APB_CLK, + "gout_hsi0_usb31drd_i_usbpcs_apb_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBPCS_APB_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_USBDPPHY_I_ACLK, + "gout_hsi0_usb31drd_usbdpphy_i_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_I_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_USBDPPHY_UDBG_I_APB_PCLK, + "gout_hsi0_usb31drd_usbdpphy_udbg_i_apb_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_UDBG_I_APB_PCLK, + 21, 0, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_XIU_D0_HSI0_ACLK, + "gout_hsi0_xiu_d0_hsi0_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D0_HSI0_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_XIU_D1_HSI0_ACLK, + "gout_hsi0_xiu_d1_hsi0_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D1_HSI0_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_XIU_P_HSI0_ACLK, + "gout_hsi0_xiu_p_hsi0_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_P_HSI0_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), +}; + +static const struct samsung_fixed_rate_clock hsi0_fixed_clks[] __initconst = { + FRATE(0, "tcxo_hsi1_hsi0", NULL, 0, 26000000), + FRATE(0, "usb20phy_phy_clock", NULL, 0, 120000000), + /* until we implement APMGSA */ + FRATE(0, "ioclk_clk_hsi0_alt", NULL, 0, 213000000), +}; + +static const struct samsung_cmu_info hsi0_cmu_info __initconst = { + .pll_clks = cmu_hsi0_pll_clks, + .nr_pll_clks = ARRAY_SIZE(cmu_hsi0_pll_clks), + .mux_clks = hsi0_mux_clks, + .nr_mux_clks = ARRAY_SIZE(hsi0_mux_clks), + .div_clks = hsi0_div_clks, + .nr_div_clks = ARRAY_SIZE(hsi0_div_clks), + .gate_clks = hsi0_gate_clks, + .nr_gate_clks = ARRAY_SIZE(hsi0_gate_clks), + .fixed_clks = hsi0_fixed_clks, + .nr_fixed_clks = ARRAY_SIZE(hsi0_fixed_clks), + .nr_clk_ids = CLKS_NR_HSI0, + .clk_regs = hsi0_clk_regs, + .nr_clk_regs = ARRAY_SIZE(hsi0_clk_regs), + .clk_name = "bus", +}; + +/* ---- CMU_HSI2 ------------------------------------------------------------ */ + +/* Register Offset definitions for CMU_HSI2 (0x14400000) */ +#define PLL_CON0_MUX_CLKCMU_HSI2_BUS_USER 0x0600 +#define PLL_CON1_MUX_CLKCMU_HSI2_BUS_USER 0x0604 +#define PLL_CON0_MUX_CLKCMU_HSI2_MMC_CARD_USER 0x0610 +#define PLL_CON1_MUX_CLKCMU_HSI2_MMC_CARD_USER 0x0614 +#define PLL_CON0_MUX_CLKCMU_HSI2_PCIE_USER 0x0620 +#define PLL_CON1_MUX_CLKCMU_HSI2_PCIE_USER 0x0624 +#define PLL_CON0_MUX_CLKCMU_HSI2_UFS_EMBD_USER 0x0630 +#define PLL_CON1_MUX_CLKCMU_HSI2_UFS_EMBD_USER 0x0634 +#define HSI2_CMU_HSI2_CONTROLLER_OPTION 0x0800 +#define CLKOUT_CON_BLK_HSI2_CMU_HSI2_CLKOUT0 0x0810 +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN 0x2000 +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN 0x2004 +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_ACLK 0x2008 +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_PCLK 0x200c +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_ACLK 0x2010 +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_PCLK 0x2014 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_D_TZPC_HSI2_IPCLKPORT_PCLK 0x201c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPC_HSI2_IPCLKPORT_PCLK 0x2020 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPIO_HSI2_IPCLKPORT_PCLK 0x2024 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_HSI2_CMU_HSI2_IPCLKPORT_PCLK 0x2028 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHM_AXI_P_HSI2_IPCLKPORT_I_CLK 0x202c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHS_ACEL_D_HSI2_IPCLKPORT_I_CLK 0x2030 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_I_ACLK 0x2034 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_SDCLKIN 0x2038 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG 0x203c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG 0x2040 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG 0x2044 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK 0x2048 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG 0x204c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG 0x2050 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG 0x2054 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK 0x2058 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PHY_UDBG_I_APB_PCLK 0x205c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PIPE_PAL_PCIE_INST_0_I_APB_PCLK 0x2060 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_SF_PCIEPHY210X2_LN05LPE_QCH_TM_WRAPPER_INST_0_I_APB_PCLK 0x2064 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4A_1_IPCLKPORT_I_CLK 0x2068 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4B_1_IPCLKPORT_I_CLK 0x206c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_ACLK 0x2070 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_PCLK 0x2074 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_ACLK 0x2078 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_PCLK 0x207c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_ACLK 0x2080 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_PCLK 0x2084 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_ACLK 0x2088 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_PCLK 0x208c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_ACLK 0x2090 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_PCLK 0x2094 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_BUS_IPCLKPORT_CLK 0x2098 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_OSCCLK_IPCLKPORT_CLK 0x209c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_ACLK 0x20a0 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_PCLK 0x20a4 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSMMU_HSI2_IPCLKPORT_CLK_S2 0x20a8 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSREG_HSI2_IPCLKPORT_PCLK 0x20ac +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_ACLK 0x20b0 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_PCLK 0x20b4 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_ACLK 0x20b8 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_PCLK 0x20bc +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_ACLK 0x20c0 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_PCLK 0x20c4 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_ACLK 0x20c8 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_PCLK 0x20cc +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_ACLK 0x20d0 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_CLK_UNIPRO 0x20d4 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_FMP_CLK 0x20d8 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_D_HSI2_IPCLKPORT_ACLK 0x20dc +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_P_HSI2_IPCLKPORT_ACLK 0x20e0 +#define DMYQCH_CON_PCIE_GEN4_1_QCH_SCLK_1 0x3000 +#define PCH_CON_LHM_AXI_P_HSI2_PCH 0x3008 +#define PCH_CON_LHS_ACEL_D_HSI2_PCH 0x300c +#define QCH_CON_D_TZPC_HSI2_QCH 0x3010 +#define QCH_CON_GPC_HSI2_QCH 0x3014 +#define QCH_CON_GPIO_HSI2_QCH 0x3018 +#define QCH_CON_HSI2_CMU_HSI2_QCH 0x301c +#define QCH_CON_LHM_AXI_P_HSI2_QCH 0x3020 +#define QCH_CON_LHS_ACEL_D_HSI2_QCH 0x3024 +#define QCH_CON_MMC_CARD_QCH 0x3028 +#define QCH_CON_PCIE_GEN4_1_QCH_APB_1 0x302c +#define QCH_CON_PCIE_GEN4_1_QCH_APB_2 0x3030 +#define QCH_CON_PCIE_GEN4_1_QCH_AXI_1 0x3034 +#define QCH_CON_PCIE_GEN4_1_QCH_AXI_2 0x3038 +#define QCH_CON_PCIE_GEN4_1_QCH_DBG_1 0x303c +#define QCH_CON_PCIE_GEN4_1_QCH_DBG_2 0x3040 +#define QCH_CON_PCIE_GEN4_1_QCH_PCS_APB 0x3044 +#define QCH_CON_PCIE_GEN4_1_QCH_PMA_APB 0x3048 +#define QCH_CON_PCIE_GEN4_1_QCH_UDBG 0x304c +#define QCH_CON_PCIE_IA_GEN4A_1_QCH 0x3050 +#define QCH_CON_PCIE_IA_GEN4B_1_QCH 0x3054 +#define QCH_CON_PPMU_HSI2_QCH 0x3058 +#define QCH_CON_QE_MMC_CARD_HSI2_QCH 0x305c +#define QCH_CON_QE_PCIE_GEN4A_HSI2_QCH 0x3060 +#define QCH_CON_QE_PCIE_GEN4B_HSI2_QCH 0x3064 +#define QCH_CON_QE_UFS_EMBD_HSI2_QCH 0x3068 +#define QCH_CON_SSMT_HSI2_QCH 0x306c +#define QCH_CON_SSMT_PCIE_IA_GEN4A_1_QCH 0x3070 +#define QCH_CON_SSMT_PCIE_IA_GEN4B_1_QCH 0x3074 +#define QCH_CON_SYSMMU_HSI2_QCH 0x3078 +#define QCH_CON_SYSREG_HSI2_QCH 0x307c +#define QCH_CON_UASC_PCIE_GEN4A_DBI_1_QCH 0x3080 +#define QCH_CON_UASC_PCIE_GEN4A_SLV_1_QCH 0x3084 +#define QCH_CON_UASC_PCIE_GEN4B_DBI_1_QCH 0x3088 +#define QCH_CON_UASC_PCIE_GEN4B_SLV_1_QCH 0x308c +#define QCH_CON_UFS_EMBD_QCH 0x3090 +#define QCH_CON_UFS_EMBD_QCH_FMP 0x3094 +#define QUEUE_CTRL_REG_BLK_HSI2_CMU_HSI2 0x3c00 + +static const unsigned long cmu_hsi2_clk_regs[] __initconst = { + PLL_CON0_MUX_CLKCMU_HSI2_BUS_USER, + PLL_CON1_MUX_CLKCMU_HSI2_BUS_USER, + PLL_CON0_MUX_CLKCMU_HSI2_MMC_CARD_USER, + PLL_CON1_MUX_CLKCMU_HSI2_MMC_CARD_USER, + PLL_CON0_MUX_CLKCMU_HSI2_PCIE_USER, + PLL_CON1_MUX_CLKCMU_HSI2_PCIE_USER, + PLL_CON0_MUX_CLKCMU_HSI2_UFS_EMBD_USER, + PLL_CON1_MUX_CLKCMU_HSI2_UFS_EMBD_USER, + HSI2_CMU_HSI2_CONTROLLER_OPTION, + CLKOUT_CON_BLK_HSI2_CMU_HSI2_CLKOUT0, + CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN, + CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN, + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_ACLK, + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_PCLK, + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_ACLK, + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_D_TZPC_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPC_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPIO_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_HSI2_CMU_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHM_AXI_P_HSI2_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHS_ACEL_D_HSI2_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_I_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_SDCLKIN, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PHY_UDBG_I_APB_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PIPE_PAL_PCIE_INST_0_I_APB_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_SF_PCIEPHY210X2_LN05LPE_QCH_TM_WRAPPER_INST_0_I_APB_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4A_1_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4B_1_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_BUS_IPCLKPORT_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_OSCCLK_IPCLKPORT_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSMMU_HSI2_IPCLKPORT_CLK_S2, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSREG_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_CLK_UNIPRO, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_FMP_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_D_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_P_HSI2_IPCLKPORT_ACLK, + DMYQCH_CON_PCIE_GEN4_1_QCH_SCLK_1, + PCH_CON_LHM_AXI_P_HSI2_PCH, + PCH_CON_LHS_ACEL_D_HSI2_PCH, + QCH_CON_D_TZPC_HSI2_QCH, + QCH_CON_GPC_HSI2_QCH, + QCH_CON_GPIO_HSI2_QCH, + QCH_CON_HSI2_CMU_HSI2_QCH, + QCH_CON_LHM_AXI_P_HSI2_QCH, + QCH_CON_LHS_ACEL_D_HSI2_QCH, + QCH_CON_MMC_CARD_QCH, + QCH_CON_PCIE_GEN4_1_QCH_APB_1, + QCH_CON_PCIE_GEN4_1_QCH_APB_2, + QCH_CON_PCIE_GEN4_1_QCH_AXI_1, + QCH_CON_PCIE_GEN4_1_QCH_AXI_2, + QCH_CON_PCIE_GEN4_1_QCH_DBG_1, + QCH_CON_PCIE_GEN4_1_QCH_DBG_2, + QCH_CON_PCIE_GEN4_1_QCH_PCS_APB, + QCH_CON_PCIE_GEN4_1_QCH_PMA_APB, + QCH_CON_PCIE_GEN4_1_QCH_UDBG, + QCH_CON_PCIE_IA_GEN4A_1_QCH, + QCH_CON_PCIE_IA_GEN4B_1_QCH, + QCH_CON_PPMU_HSI2_QCH, + QCH_CON_QE_MMC_CARD_HSI2_QCH, + QCH_CON_QE_PCIE_GEN4A_HSI2_QCH, + QCH_CON_QE_PCIE_GEN4B_HSI2_QCH, + QCH_CON_QE_UFS_EMBD_HSI2_QCH, + QCH_CON_SSMT_HSI2_QCH, + QCH_CON_SSMT_PCIE_IA_GEN4A_1_QCH, + QCH_CON_SSMT_PCIE_IA_GEN4B_1_QCH, + QCH_CON_SYSMMU_HSI2_QCH, + QCH_CON_SYSREG_HSI2_QCH, + QCH_CON_UASC_PCIE_GEN4A_DBI_1_QCH, + QCH_CON_UASC_PCIE_GEN4A_SLV_1_QCH, + QCH_CON_UASC_PCIE_GEN4B_DBI_1_QCH, + QCH_CON_UASC_PCIE_GEN4B_SLV_1_QCH, + QCH_CON_UFS_EMBD_QCH, + QCH_CON_UFS_EMBD_QCH_FMP, + QUEUE_CTRL_REG_BLK_HSI2_CMU_HSI2, +}; + +PNAME(mout_hsi2_bus_user_p) = { "oscclk", "dout_cmu_hsi2_bus" }; +PNAME(mout_hsi2_mmc_card_user_p) = { "oscclk", "dout_cmu_hsi2_mmc_card" }; +PNAME(mout_hsi2_pcie_user_p) = { "oscclk", "dout_cmu_hsi2_pcie" }; +PNAME(mout_hsi2_ufs_embd_user_p) = { "oscclk", "dout_cmu_hsi2_ufs_embd" }; + +static const struct samsung_mux_clock hsi2_mux_clks[] __initconst = { + MUX(CLK_MOUT_HSI2_BUS_USER, "mout_hsi2_bus_user", mout_hsi2_bus_user_p, + PLL_CON0_MUX_CLKCMU_HSI2_BUS_USER, 4, 1), + MUX(CLK_MOUT_HSI2_MMC_CARD_USER, "mout_hsi2_mmc_card_user", + mout_hsi2_mmc_card_user_p, PLL_CON0_MUX_CLKCMU_HSI2_MMC_CARD_USER, + 4, 1), + MUX(CLK_MOUT_HSI2_PCIE_USER, "mout_hsi2_pcie_user", + mout_hsi2_pcie_user_p, PLL_CON0_MUX_CLKCMU_HSI2_PCIE_USER, + 4, 1), + MUX(CLK_MOUT_HSI2_UFS_EMBD_USER, "mout_hsi2_ufs_embd_user", + mout_hsi2_ufs_embd_user_p, PLL_CON0_MUX_CLKCMU_HSI2_UFS_EMBD_USER, + 4, 1), +}; + +static const struct samsung_gate_clock hsi2_gate_clks[] __initconst = { + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_003_PHY_REFCLK_IN, + "gout_hsi2_pcie_gen4_1_pcie_003_phy_refclk_in", + "mout_hsi2_pcie_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_004_PHY_REFCLK_IN, + "gout_hsi2_pcie_gen4_1_pcie_004_phy_refclk_in", + "mout_hsi2_pcie_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_SSMT_PCIE_IA_GEN4A_1_ACLK, + "gout_hsi2_ssmt_pcie_ia_gen4a_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_SSMT_PCIE_IA_GEN4A_1_PCLK, + "gout_hsi2_ssmt_pcie_ia_gen4a_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_SSMT_PCIE_IA_GEN4B_1_ACLK, + "gout_hsi2_ssmt_pcie_ia_gen4b_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_SSMT_PCIE_IA_GEN4B_1_PCLK, + "gout_hsi2_ssmt_pcie_ia_gen4b_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_D_TZPC_HSI2_PCLK, + "gout_hsi2_d_tzpc_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_D_TZPC_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_GPC_HSI2_PCLK, + "gout_hsi2_gpc_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPC_HSI2_IPCLKPORT_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI2_GPIO_HSI2_PCLK, + "gout_hsi2_gpio_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPIO_HSI2_IPCLKPORT_PCLK, 21, + CLK_IGNORE_UNUSED, 0), + /* Disabling this clock makes the system hang. Mark the clock as critical. */ + GATE(CLK_GOUT_HSI2_HSI2_CMU_HSI2_PCLK, + "gout_hsi2_hsi2_cmu_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_HSI2_CMU_HSI2_IPCLKPORT_PCLK, + 21, CLK_IS_CRITICAL, 0), + /* Disabling this clock makes the system hang. Mark the clock as critical. */ + GATE(CLK_GOUT_HSI2_LHM_AXI_P_HSI2_I_CLK, + "gout_hsi2_lhm_axi_p_hsi2_i_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHM_AXI_P_HSI2_IPCLKPORT_I_CLK, + 21, CLK_IS_CRITICAL, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_LHS_ACEL_D_HSI2_I_CLK, + "gout_hsi2_lhs_acel_d_hsi2_i_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHS_ACEL_D_HSI2_IPCLKPORT_I_CLK, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI2_MMC_CARD_I_ACLK, + "gout_hsi2_mmc_card_i_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_I_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_MMC_CARD_SDCLKIN, + "gout_hsi2_mmc_card_sdclkin", "mout_hsi2_mmc_card_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_SDCLKIN, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_003_DBI_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_003_dbi_aclk_ug", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_003_MSTR_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_003_mstr_aclk_ug", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_003_SLV_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_003_slv_aclk_ug", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_003_I_DRIVER_APB_CLK, + "gout_hsi2_pcie_gen4_1_pcie_003_i_driver_apb_clk", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_004_DBI_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_004_dbi_aclk_ug", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_004_MSTR_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_004_mstr_aclk_ug", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_004_SLV_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_004_slv_aclk_ug", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_004_I_DRIVER_APB_CLK, + "gout_hsi2_pcie_gen4_1_pcie_004_i_driver_apb_clk", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCS_PMA_PHY_UDBG_I_APB_PCLK, + "gout_hsi2_pcie_gen4_1_pcs_pma_phy_udbg_i_apb_pclk", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PHY_UDBG_I_APB_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCS_PMA_PIPE_PAL_PCIE_I_APB_PCLK, + "gout_hsi2_pcie_gen4_1_pcs_pma_pipe_pal_pcie_i_apb_pclk", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PIPE_PAL_PCIE_INST_0_I_APB_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCS_PMA_PCIEPHY210X2_QCH_I_APB_PCLK, + "gout_hsi2_pcie_gen4_1_pcs_pma_pciephy210x2_qch_i_apb_pclk", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_SF_PCIEPHY210X2_LN05LPE_QCH_TM_WRAPPER_INST_0_I_APB_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_IA_GEN4A_1_I_CLK, + "gout_hsi2_pcie_ia_gen4a_1_i_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4A_1_IPCLKPORT_I_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_IA_GEN4B_1_I_CLK, + "gout_hsi2_pcie_ia_gen4b_1_i_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4B_1_IPCLKPORT_I_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PPMU_HSI2_ACLK, + "gout_hsi2_ppmu_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PPMU_HSI2_PCLK, + "gout_hsi2_ppmu_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_MMC_CARD_HSI2_ACLK, + "gout_hsi2_qe_mmc_card_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_MMC_CARD_HSI2_PCLK, + "gout_hsi2_qe_mmc_card_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_PCIE_GEN4A_HSI2_ACLK, + "gout_hsi2_qe_pcie_gen4a_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_PCIE_GEN4A_HSI2_PCLK, + "gout_hsi2_qe_pcie_gen4a_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_PCIE_GEN4B_HSI2_ACLK, + "gout_hsi2_qe_pcie_gen4b_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_PCIE_GEN4B_HSI2_PCLK, + "gout_hsi2_qe_pcie_gen4b_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_UFS_EMBD_HSI2_ACLK, + "gout_hsi2_qe_ufs_embd_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_UFS_EMBD_HSI2_PCLK, + "gout_hsi2_qe_ufs_embd_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_CLK_HSI2_BUS_CLK, + "gout_hsi2_clk_hsi2_bus_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_BUS_IPCLKPORT_CLK, + 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_HSI2_CLK_HSI2_OSCCLK_CLK, + "gout_hsi2_clk_hsi2_oscclk_clk", "oscclk", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_OSCCLK_IPCLKPORT_CLK, + 21, 0, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_SSMT_HSI2_ACLK, + "gout_hsi2_ssmt_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_SSMT_HSI2_PCLK, + "gout_hsi2_ssmt_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_PCLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_SYSMMU_HSI2_CLK_S2, + "gout_hsi2_sysmmu_hsi2_clk_s2", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSMMU_HSI2_IPCLKPORT_CLK_S2, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI2_SYSREG_HSI2_PCLK, + "gout_hsi2_sysreg_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSREG_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4A_DBI_1_ACLK, + "gout_hsi2_uasc_pcie_gen4a_dbi_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4A_DBI_1_PCLK, + "gout_hsi2_uasc_pcie_gen4a_dbi_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4A_SLV_1_ACLK, + "gout_hsi2_uasc_pcie_gen4a_slv_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4A_SLV_1_PCLK, + "gout_hsi2_uasc_pcie_gen4a_slv_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4B_DBI_1_ACLK, + "gout_hsi2_uasc_pcie_gen4b_dbi_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4B_DBI_1_PCLK, + "gout_hsi2_uasc_pcie_gen4b_dbi_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4B_SLV_1_ACLK, + "gout_hsi2_uasc_pcie_gen4b_slv_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4B_SLV_1_PCLK, + "gout_hsi2_uasc_pcie_gen4b_slv_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UFS_EMBD_I_ACLK, + "gout_hsi2_ufs_embd_i_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UFS_EMBD_I_CLK_UNIPRO, + "gout_hsi2_ufs_embd_i_clk_unipro", "mout_hsi2_ufs_embd_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_CLK_UNIPRO, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UFS_EMBD_I_FMP_CLK, + "gout_hsi2_ufs_embd_i_fmp_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_FMP_CLK, + 21, 0, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_XIU_D_HSI2_ACLK, + "gout_hsi2_xiu_d_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_D_HSI2_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_XIU_P_HSI2_ACLK, + "gout_hsi2_xiu_p_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_P_HSI2_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), +}; + +static const struct samsung_cmu_info hsi2_cmu_info __initconst = { + .mux_clks = hsi2_mux_clks, + .nr_mux_clks = ARRAY_SIZE(hsi2_mux_clks), + .gate_clks = hsi2_gate_clks, + .nr_gate_clks = ARRAY_SIZE(hsi2_gate_clks), + .nr_clk_ids = CLKS_NR_HSI2, + .clk_regs = cmu_hsi2_clk_regs, + .nr_clk_regs = ARRAY_SIZE(cmu_hsi2_clk_regs), + .clk_name = "bus", +}; + /* ---- CMU_MISC ------------------------------------------------------------ */ /* Register Offset definitions for CMU_MISC (0x10010000) */ @@ -2763,33 +3718,33 @@ static const struct samsung_mux_clock peric0_mux_clks[] __initconst = { MUX(CLK_MOUT_PERIC0_USI0_UART_USER, "mout_peric0_usi0_uart_user", mout_peric0_usi0_uart_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI0_UART_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI14_USI_USER, - "mout_peric0_usi14_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI14_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI1_USI_USER, - "mout_peric0_usi1_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI1_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI2_USI_USER, - "mout_peric0_usi2_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI2_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI3_USI_USER, - "mout_peric0_usi3_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI3_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI4_USI_USER, - "mout_peric0_usi4_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI4_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI5_USI_USER, - "mout_peric0_usi5_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI5_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI6_USI_USER, - "mout_peric0_usi6_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI6_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI7_USI_USER, - "mout_peric0_usi7_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI7_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI8_USI_USER, - "mout_peric0_usi8_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI8_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI14_USI_USER, + "mout_peric0_usi14_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI14_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI1_USI_USER, + "mout_peric0_usi1_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI1_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI2_USI_USER, + "mout_peric0_usi2_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI2_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI3_USI_USER, + "mout_peric0_usi3_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI3_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI4_USI_USER, + "mout_peric0_usi4_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI4_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI5_USI_USER, + "mout_peric0_usi5_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI5_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI6_USI_USER, + "mout_peric0_usi6_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI6_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI7_USI_USER, + "mout_peric0_usi7_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI7_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI8_USI_USER, + "mout_peric0_usi8_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI8_USI_USER, 4, 1), }; static const struct samsung_div_clock peric0_div_clks[] __initconst = { @@ -2798,33 +3753,42 @@ static const struct samsung_div_clock peric0_div_clks[] __initconst = { DIV(CLK_DOUT_PERIC0_USI0_UART, "dout_peric0_usi0_uart", "mout_peric0_usi0_uart_user", CLK_CON_DIV_DIV_CLK_PERIC0_USI0_UART, 0, 4), - DIV(CLK_DOUT_PERIC0_USI14_USI, - "dout_peric0_usi14_usi", "mout_peric0_usi14_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI14_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI1_USI, - "dout_peric0_usi1_usi", "mout_peric0_usi1_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI1_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI2_USI, - "dout_peric0_usi2_usi", "mout_peric0_usi2_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI2_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI3_USI, - "dout_peric0_usi3_usi", "mout_peric0_usi3_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI3_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI4_USI, - "dout_peric0_usi4_usi", "mout_peric0_usi4_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI4_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI5_USI, - "dout_peric0_usi5_usi", "mout_peric0_usi5_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI5_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI6_USI, - "dout_peric0_usi6_usi", "mout_peric0_usi6_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI6_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI7_USI, - "dout_peric0_usi7_usi", "mout_peric0_usi7_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI7_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI8_USI, - "dout_peric0_usi8_usi", "mout_peric0_usi8_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI8_USI, 0, 4), + DIV_F(CLK_DOUT_PERIC0_USI14_USI, + "dout_peric0_usi14_usi", "mout_peric0_usi14_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI14_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI1_USI, + "dout_peric0_usi1_usi", "mout_peric0_usi1_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI1_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI2_USI, + "dout_peric0_usi2_usi", "mout_peric0_usi2_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI2_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI3_USI, + "dout_peric0_usi3_usi", "mout_peric0_usi3_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI3_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI4_USI, + "dout_peric0_usi4_usi", "mout_peric0_usi4_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI4_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI5_USI, + "dout_peric0_usi5_usi", "mout_peric0_usi5_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI5_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI6_USI, + "dout_peric0_usi6_usi", "mout_peric0_usi6_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI6_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI7_USI, + "dout_peric0_usi7_usi", "mout_peric0_usi7_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI7_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI8_USI, + "dout_peric0_usi8_usi", "mout_peric0_usi8_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI8_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), }; static const struct samsung_gate_clock peric0_gate_clks[] __initconst = { @@ -2857,11 +3821,11 @@ static const struct samsung_gate_clock peric0_gate_clks[] __initconst = { GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_0, "gout_peric0_peric0_top0_ipclk_0", "dout_peric0_usi1_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_0, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_1, "gout_peric0_peric0_top0_ipclk_1", "dout_peric0_usi2_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_1, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_10, "gout_peric0_peric0_top0_ipclk_10", "dout_peric0_i3c", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_10, @@ -2889,27 +3853,27 @@ static const struct samsung_gate_clock peric0_gate_clks[] __initconst = { GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_2, "gout_peric0_peric0_top0_ipclk_2", "dout_peric0_usi3_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_2, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_3, "gout_peric0_peric0_top0_ipclk_3", "dout_peric0_usi4_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_3, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_4, "gout_peric0_peric0_top0_ipclk_4", "dout_peric0_usi5_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_4, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_5, "gout_peric0_peric0_top0_ipclk_5", "dout_peric0_usi6_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_5, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_6, "gout_peric0_peric0_top0_ipclk_6", "dout_peric0_usi7_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_6, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_7, "gout_peric0_peric0_top0_ipclk_7", "dout_peric0_usi8_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_7, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_8, "gout_peric0_peric0_top0_ipclk_8", "dout_peric0_i3c", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_8, @@ -2990,7 +3954,7 @@ static const struct samsung_gate_clock peric0_gate_clks[] __initconst = { GATE(CLK_GOUT_PERIC0_PERIC0_TOP1_IPCLK_2, "gout_peric0_peric0_top1_ipclk_2", "dout_peric0_usi14_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP1_IPCLKPORT_IPCLK_2, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), /* Disabling this clock makes the system hang. Mark the clock as critical. */ GATE(CLK_GOUT_PERIC0_PERIC0_TOP1_PCLK_0, "gout_peric0_peric0_top1_pclk_0", "mout_peric0_bus_user", @@ -3230,47 +4194,53 @@ static const struct samsung_mux_clock peric1_mux_clks[] __initconst = { MUX(CLK_MOUT_PERIC1_I3C_USER, "mout_peric1_i3c_user", mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_I3C_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI0_USI_USER, - "mout_peric1_usi0_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI0_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI10_USI_USER, - "mout_peric1_usi10_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI10_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI11_USI_USER, - "mout_peric1_usi11_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI11_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI12_USI_USER, - "mout_peric1_usi12_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI12_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI13_USI_USER, - "mout_peric1_usi13_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI13_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI9_USI_USER, - "mout_peric1_usi9_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI9_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI0_USI_USER, + "mout_peric1_usi0_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI0_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI10_USI_USER, + "mout_peric1_usi10_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI10_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI11_USI_USER, + "mout_peric1_usi11_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI11_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI12_USI_USER, + "mout_peric1_usi12_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI12_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI13_USI_USER, + "mout_peric1_usi13_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI13_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI9_USI_USER, + "mout_peric1_usi9_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI9_USI_USER, 4, 1), }; static const struct samsung_div_clock peric1_div_clks[] __initconst = { DIV(CLK_DOUT_PERIC1_I3C, "dout_peric1_i3c", "mout_peric1_i3c_user", CLK_CON_DIV_DIV_CLK_PERIC1_I3C, 0, 4), - DIV(CLK_DOUT_PERIC1_USI0_USI, - "dout_peric1_usi0_usi", "mout_peric1_usi0_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI0_USI, 0, 4), - DIV(CLK_DOUT_PERIC1_USI10_USI, - "dout_peric1_usi10_usi", "mout_peric1_usi10_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI10_USI, 0, 4), - DIV(CLK_DOUT_PERIC1_USI11_USI, - "dout_peric1_usi11_usi", "mout_peric1_usi11_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI11_USI, 0, 4), - DIV(CLK_DOUT_PERIC1_USI12_USI, - "dout_peric1_usi12_usi", "mout_peric1_usi12_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI12_USI, 0, 4), - DIV(CLK_DOUT_PERIC1_USI13_USI, - "dout_peric1_usi13_usi", "mout_peric1_usi13_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI13_USI, 0, 4), - DIV(CLK_DOUT_PERIC1_USI9_USI, - "dout_peric1_usi9_usi", "mout_peric1_usi9_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI9_USI, 0, 4), + DIV_F(CLK_DOUT_PERIC1_USI0_USI, + "dout_peric1_usi0_usi", "mout_peric1_usi0_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI0_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI10_USI, + "dout_peric1_usi10_usi", "mout_peric1_usi10_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI10_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI11_USI, + "dout_peric1_usi11_usi", "mout_peric1_usi11_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI11_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI12_USI, + "dout_peric1_usi12_usi", "mout_peric1_usi12_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI12_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI13_USI, + "dout_peric1_usi13_usi", "mout_peric1_usi13_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI13_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI9_USI, + "dout_peric1_usi9_usi", "mout_peric1_usi9_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI9_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), }; static const struct samsung_gate_clock peric1_gate_clks[] __initconst = { @@ -3305,27 +4275,27 @@ static const struct samsung_gate_clock peric1_gate_clks[] __initconst = { GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_1, "gout_peric1_peric1_top0_ipclk_1", "dout_peric1_usi0_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_1, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_2, "gout_peric1_peric1_top0_ipclk_2", "dout_peric1_usi9_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_2, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_3, "gout_peric1_peric1_top0_ipclk_3", "dout_peric1_usi10_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_3, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_4, "gout_peric1_peric1_top0_ipclk_4", "dout_peric1_usi11_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_4, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_5, "gout_peric1_peric1_top0_ipclk_5", "dout_peric1_usi12_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_5, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_6, "gout_peric1_peric1_top0_ipclk_6", "dout_peric1_usi13_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_6, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_8, "gout_peric1_peric1_top0_ipclk_8", "dout_peric1_i3c", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_8, @@ -3427,6 +4397,12 @@ static const struct of_device_id gs101_cmu_of_match[] = { .compatible = "google,gs101-cmu-apm", .data = &apm_cmu_info, }, { + .compatible = "google,gs101-cmu-hsi0", + .data = &hsi0_cmu_info, + }, { + .compatible = "google,gs101-cmu-hsi2", + .data = &hsi2_cmu_info, + }, { .compatible = "google,gs101-cmu-peric0", .data = &peric0_cmu_info, }, { diff --git a/drivers/clk/samsung/clk.c b/drivers/clk/samsung/clk.c index b6701905f254..afa5760ed3a1 100644 --- a/drivers/clk/samsung/clk.c +++ b/drivers/clk/samsung/clk.c @@ -139,7 +139,7 @@ void __init samsung_clk_register_fixed_rate(struct samsung_clk_provider *ctx, unsigned int nr_clk) { struct clk_hw *clk_hw; - unsigned int idx, ret; + unsigned int idx; for (idx = 0; idx < nr_clk; idx++, list++) { clk_hw = clk_hw_register_fixed_rate(ctx->dev, list->name, @@ -151,15 +151,6 @@ void __init samsung_clk_register_fixed_rate(struct samsung_clk_provider *ctx, } samsung_clk_add_lookup(ctx, clk_hw, list->id); - - /* - * Unconditionally add a clock lookup for the fixed rate clocks. - * There are not many of these on any of Samsung platforms. - */ - ret = clk_hw_register_clkdev(clk_hw, list->name, NULL); - if (ret) - pr_err("%s: failed to register clock lookup for %s", - __func__, list->name); } } diff --git a/drivers/clk/samsung/clk.h b/drivers/clk/samsung/clk.h index a763309e6f12..fb06caa71f0a 100644 --- a/drivers/clk/samsung/clk.h +++ b/drivers/clk/samsung/clk.h @@ -133,7 +133,7 @@ struct samsung_mux_clock { .name = cname, \ .parent_names = pnames, \ .num_parents = ARRAY_SIZE(pnames), \ - .flags = (f) | CLK_SET_RATE_NO_REPARENT, \ + .flags = f, \ .offset = o, \ .shift = s, \ .width = w, \ @@ -141,9 +141,16 @@ struct samsung_mux_clock { } #define MUX(_id, cname, pnames, o, s, w) \ - __MUX(_id, cname, pnames, o, s, w, 0, 0) + __MUX(_id, cname, pnames, o, s, w, CLK_SET_RATE_NO_REPARENT, 0) #define MUX_F(_id, cname, pnames, o, s, w, f, mf) \ + __MUX(_id, cname, pnames, o, s, w, (f) | CLK_SET_RATE_NO_REPARENT, mf) + +/* Used by MUX clocks where reparenting on clock rate change is allowed. */ +#define nMUX(_id, cname, pnames, o, s, w) \ + __MUX(_id, cname, pnames, o, s, w, 0, 0) + +#define nMUX_F(_id, cname, pnames, o, s, w, f, mf) \ __MUX(_id, cname, pnames, o, s, w, f, mf) /** @@ -330,6 +337,7 @@ struct samsung_clock_reg_cache { * @suspend_regs: list of clock registers to set before suspend * @nr_suspend_regs: count of clock registers in @suspend_regs * @clk_name: name of the parent clock needed for CMU register access + * @manual_plls: Enable manual control for PLL clocks */ struct samsung_cmu_info { const struct samsung_pll_clock *pll_clks; @@ -354,6 +362,9 @@ struct samsung_cmu_info { const struct samsung_clk_reg_dump *suspend_regs; unsigned int nr_suspend_regs; const char *clk_name; + + /* ARM64 Exynos CMUs */ + bool manual_plls; }; struct samsung_clk_provider *samsung_clk_init(struct device *dev, diff --git a/drivers/clk/sophgo/Kconfig b/drivers/clk/sophgo/Kconfig new file mode 100644 index 000000000000..1cc49be71bdb --- /dev/null +++ b/drivers/clk/sophgo/Kconfig @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +# common clock support for SOPHGO SoC family. + +config CLK_SOPHGO_CV1800 + tristate "Support for the Sophgo CV1800 series SoCs clock controller" + depends on ARCH_SOPHGO || COMPILE_TEST + help + This driver supports clock controller of Sophgo CV18XX series SoC. + The driver require a 25MHz Oscillator to function generate clock. + It includes PLLs, common clock function and some vendor clock for + IPs of CV18XX series SoC diff --git a/drivers/clk/sophgo/Makefile b/drivers/clk/sophgo/Makefile new file mode 100644 index 000000000000..a50320764200 --- /dev/null +++ b/drivers/clk/sophgo/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CLK_SOPHGO_CV1800) += clk-sophgo-cv1800.o + +clk-sophgo-cv1800-y += clk-cv1800.o +clk-sophgo-cv1800-y += clk-cv18xx-common.o +clk-sophgo-cv1800-y += clk-cv18xx-ip.o +clk-sophgo-cv1800-y += clk-cv18xx-pll.o diff --git a/drivers/clk/sophgo/clk-cv1800.c b/drivers/clk/sophgo/clk-cv1800.c new file mode 100644 index 000000000000..2da4c24621cf --- /dev/null +++ b/drivers/clk/sophgo/clk-cv1800.c @@ -0,0 +1,1537 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Inochi Amaoto <inochiama@outlook.com> + */ + +#include <linux/module.h> +#include <linux/clk-provider.h> +#include <linux/platform_device.h> +#include <linux/io.h> +#include <linux/spinlock.h> + +#include "clk-cv1800.h" + +#include "clk-cv18xx-common.h" +#include "clk-cv18xx-ip.h" +#include "clk-cv18xx-pll.h" + +struct cv1800_clk_ctrl; + +struct cv1800_clk_desc { + struct clk_hw_onecell_data *clks_data; + + int (*pre_init)(struct device *dev, void __iomem *base, + struct cv1800_clk_ctrl *ctrl, + const struct cv1800_clk_desc *desc); +}; + +struct cv1800_clk_ctrl { + const struct cv1800_clk_desc *desc; + spinlock_t lock; +}; + +#define CV1800_DIV_FLAG \ + (CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ROUND_CLOSEST) +static const struct clk_parent_data osc_parents[] = { + { .index = 0 }, +}; + +static const struct cv1800_clk_pll_limit pll_limits[] = { + { + .pre_div = _CV1800_PLL_LIMIT(1, 127), + .div = _CV1800_PLL_LIMIT(6, 127), + .post_div = _CV1800_PLL_LIMIT(1, 127), + .ictrl = _CV1800_PLL_LIMIT(0, 7), + .mode = _CV1800_PLL_LIMIT(0, 3), + }, + { + .pre_div = _CV1800_PLL_LIMIT(1, 127), + .div = _CV1800_PLL_LIMIT(6, 127), + .post_div = _CV1800_PLL_LIMIT(1, 127), + .ictrl = _CV1800_PLL_LIMIT(0, 7), + .mode = _CV1800_PLL_LIMIT(0, 3), + }, +}; + +static CV1800_INTEGRAL_PLL(clk_fpll, osc_parents, + REG_FPLL_CSR, + REG_PLL_G6_CTRL, 8, + REG_PLL_G6_STATUS, 2, + pll_limits, + CLK_IS_CRITICAL); + +static CV1800_INTEGRAL_PLL(clk_mipimpll, osc_parents, + REG_MIPIMPLL_CSR, + REG_PLL_G2_CTRL, 0, + REG_PLL_G2_STATUS, 0, + pll_limits, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_mipimpll_parents[] = { + { .hw = &clk_mipimpll.common.hw }, +}; +static const struct clk_parent_data clk_bypass_mipimpll_parents[] = { + { .index = 0 }, + { .hw = &clk_mipimpll.common.hw }, +}; +static const struct clk_parent_data clk_bypass_fpll_parents[] = { + { .index = 0 }, + { .hw = &clk_fpll.common.hw }, +}; + +static struct cv1800_clk_pll_synthesizer clk_mpll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 2), + .clk_half = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 0), + .ctrl = REG_MPLL_SSC_SYN_CTRL, + .set = REG_MPLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_mpll, clk_bypass_mipimpll_parents, + REG_MPLL_CSR, + REG_PLL_G6_CTRL, 0, + REG_PLL_G6_STATUS, 0, + pll_limits, + &clk_mpll_synthesizer, + CLK_IS_CRITICAL); + +static struct cv1800_clk_pll_synthesizer clk_tpll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 3), + .clk_half = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 0), + .ctrl = REG_TPLL_SSC_SYN_CTRL, + .set = REG_TPLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_tpll, clk_bypass_mipimpll_parents, + REG_TPLL_CSR, + REG_PLL_G6_CTRL, 4, + REG_PLL_G6_STATUS, 1, + pll_limits, + &clk_tpll_synthesizer, + CLK_IS_CRITICAL); + +static struct cv1800_clk_pll_synthesizer clk_a0pll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 2), + .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), + .ctrl = REG_A0PLL_SSC_SYN_CTRL, + .set = REG_A0PLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_a0pll, clk_bypass_mipimpll_parents, + REG_A0PLL_CSR, + REG_PLL_G2_CTRL, 4, + REG_PLL_G2_STATUS, 1, + pll_limits, + &clk_a0pll_synthesizer, + CLK_IS_CRITICAL); + +static struct cv1800_clk_pll_synthesizer clk_disppll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 3), + .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), + .ctrl = REG_DISPPLL_SSC_SYN_CTRL, + .set = REG_DISPPLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_disppll, clk_bypass_mipimpll_parents, + REG_DISPPLL_CSR, + REG_PLL_G2_CTRL, 8, + REG_PLL_G2_STATUS, 2, + pll_limits, + &clk_disppll_synthesizer, + CLK_IS_CRITICAL); + +static struct cv1800_clk_pll_synthesizer clk_cam0pll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 4), + .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), + .ctrl = REG_CAM0PLL_SSC_SYN_CTRL, + .set = REG_CAM0PLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_cam0pll, clk_bypass_mipimpll_parents, + REG_CAM0PLL_CSR, + REG_PLL_G2_CTRL, 12, + REG_PLL_G2_STATUS, 3, + pll_limits, + &clk_cam0pll_synthesizer, + CLK_IGNORE_UNUSED); + +static struct cv1800_clk_pll_synthesizer clk_cam1pll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 5), + .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), + .ctrl = REG_CAM1PLL_SSC_SYN_CTRL, + .set = REG_CAM1PLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_cam1pll, clk_bypass_mipimpll_parents, + REG_CAM1PLL_CSR, + REG_PLL_G2_CTRL, 16, + REG_PLL_G2_STATUS, 4, + pll_limits, + &clk_cam1pll_synthesizer, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_cam0pll_parents[] = { + { .hw = &clk_cam0pll.common.hw }, +}; + +/* G2D */ +static CV1800_FIXED_DIV(clk_cam0pll_d2, clk_cam0pll_parents, + REG_CAM0PLL_CLK_CSR, 1, + 2, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED); +static CV1800_FIXED_DIV(clk_cam0pll_d3, clk_cam0pll_parents, + REG_CAM0PLL_CLK_CSR, 2, + 3, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED); +static CV1800_FIXED_DIV(clk_mipimpll_d3, clk_mipimpll_parents, + REG_MIPIMPLL_CLK_CSR, 2, + 3, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED); + +/* TPU */ +static const struct clk_parent_data clk_tpu_parents[] = { + { .index = 0 }, + { .hw = &clk_tpll.common.hw }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_tpu, clk_tpu_parents, + REG_CLK_EN_0, 4, + REG_DIV_CLK_TPU, 16, 4, 3, CV1800_DIV_FLAG, + REG_DIV_CLK_TPU, 8, 2, + REG_CLK_BYP_0, 3, + 0); +static CV1800_GATE(clk_tpu_fab, clk_mipimpll_parents, + REG_CLK_EN_0, 5, + 0); + +/* FABRIC_AXI6 */ +static CV1800_BYPASS_DIV(clk_axi6, clk_bypass_fpll_parents, + REG_CLK_EN_2, 2, + REG_DIV_CLK_AXI6, 16, 4, 15, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 20, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_axi6_bus_parents[] = { + { .hw = &clk_axi6.div.common.hw }, +}; +static const struct clk_parent_data clk_bypass_axi6_bus_parents[] = { + { .index = 0 }, + { .hw = &clk_axi6.div.common.hw }, +}; + +/* FABRIC_AXI4 */ +static const struct clk_parent_data clk_axi4_parents[] = { + { .index = 0 }, + { .hw = &clk_fpll.common.hw }, + { .hw = &clk_disppll.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_axi4, clk_axi4_parents, + REG_CLK_EN_2, 1, + REG_DIV_CLK_AXI4, 16, 4, 5, CV1800_DIV_FLAG, + REG_DIV_CLK_AXI4, 8, 2, + REG_CLK_BYP_0, 19, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_axi4_bus_parents[] = { + { .hw = &clk_axi4.mux.common.hw }, +}; + +/* XTAL_MISC */ +static CV1800_GATE(clk_xtal_misc, osc_parents, + REG_CLK_EN_0, 14, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_timer_parents[] = { + { .hw = &clk_xtal_misc.common.hw }, +}; + +/* TOP */ +static const struct clk_parent_data clk_cam0_200_parents[] = { + { .index = 0 }, + { .index = 0 }, + { .hw = &clk_disppll.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_cam0_200, clk_cam0_200_parents, + REG_CLK_EN_1, 13, + REG_DIV_CLK_CAM0_200, 16, 4, 1, CV1800_DIV_FLAG, + REG_DIV_CLK_CAM0_200, 8, 2, + REG_CLK_BYP_0, 16, + CLK_IS_CRITICAL); +static CV1800_DIV(clk_1m, osc_parents, + REG_CLK_EN_3, 5, + REG_DIV_CLK_1M, 16, 6, 25, CV1800_DIV_FLAG, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_pm, clk_axi6_bus_parents, + REG_CLK_EN_3, 8, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer0, clk_timer_parents, + REG_CLK_EN_3, 9, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer1, clk_timer_parents, + REG_CLK_EN_3, 10, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer2, clk_timer_parents, + REG_CLK_EN_3, 11, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer3, clk_timer_parents, + REG_CLK_EN_3, 12, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer4, clk_timer_parents, + REG_CLK_EN_3, 13, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer5, clk_timer_parents, + REG_CLK_EN_3, 14, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer6, clk_timer_parents, + REG_CLK_EN_3, 15, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer7, clk_timer_parents, + REG_CLK_EN_3, 16, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_parents_1m[] = { + { .hw = &clk_1m.common.hw }, +}; +static const struct clk_parent_data clk_uart_parents[] = { + { .hw = &clk_cam0_200.mux.common.hw }, +}; + +/* AHB ROM */ +static CV1800_GATE(clk_ahb_rom, clk_axi4_bus_parents, + REG_CLK_EN_0, 6, + 0); + +/* RTC */ +static CV1800_GATE(clk_rtc_25m, osc_parents, + REG_CLK_EN_0, 8, + CLK_IS_CRITICAL); +static CV1800_BYPASS_DIV(clk_src_rtc_sys_0, clk_bypass_fpll_parents, + REG_CLK_EN_4, 6, + REG_DIV_CLK_RTCSYS_SRC_0, 16, 4, 5, CV1800_DIV_FLAG, + REG_CLK_BYP_1, 5, + CLK_IS_CRITICAL); + +/* TEMPSEN */ +static CV1800_GATE(clk_tempsen, osc_parents, + REG_CLK_EN_0, 9, + 0); + +/* SARADC */ +static CV1800_GATE(clk_saradc, osc_parents, + REG_CLK_EN_0, 10, + 0); + +/* EFUSE */ +static CV1800_GATE(clk_efuse, osc_parents, + REG_CLK_EN_0, 11, + 0); +static CV1800_GATE(clk_apb_efuse, osc_parents, + REG_CLK_EN_0, 12, + 0); + +/* WDT */ +static CV1800_GATE(clk_apb_wdt, osc_parents, + REG_CLK_EN_1, 7, + CLK_IS_CRITICAL); + +/* WGN */ +static CV1800_GATE(clk_wgn, osc_parents, + REG_CLK_EN_3, 22, + 0); +static CV1800_GATE(clk_wgn0, osc_parents, + REG_CLK_EN_3, 23, + 0); +static CV1800_GATE(clk_wgn1, osc_parents, + REG_CLK_EN_3, 24, + 0); +static CV1800_GATE(clk_wgn2, osc_parents, + REG_CLK_EN_3, 25, + 0); + +/* KEYSCAN */ +static CV1800_GATE(clk_keyscan, osc_parents, + REG_CLK_EN_3, 26, + 0); + +/* EMMC */ +static CV1800_GATE(clk_axi4_emmc, clk_axi4_bus_parents, + REG_CLK_EN_0, 15, + 0); +static CV1800_BYPASS_MUX(clk_emmc, clk_axi4_parents, + REG_CLK_EN_0, 16, + REG_DIV_CLK_EMMC, 16, 5, 15, CV1800_DIV_FLAG, + REG_DIV_CLK_EMMC, 8, 2, + REG_CLK_BYP_0, 5, + 0); +static CV1800_DIV(clk_emmc_100k, clk_parents_1m, + REG_CLK_EN_0, 17, + REG_DIV_CLK_EMMC_100K, 16, 8, 10, CV1800_DIV_FLAG, + 0); + +/* SD */ +static CV1800_GATE(clk_axi4_sd0, clk_axi4_bus_parents, + REG_CLK_EN_0, 18, + 0); +static CV1800_BYPASS_MUX(clk_sd0, clk_axi4_parents, + REG_CLK_EN_0, 19, + REG_DIV_CLK_SD0, 16, 5, 15, CV1800_DIV_FLAG, + REG_DIV_CLK_SD0, 8, 2, + REG_CLK_BYP_0, 6, + 0); +static CV1800_DIV(clk_sd0_100k, clk_parents_1m, + REG_CLK_EN_0, 20, + REG_DIV_CLK_SD0_100K, 16, 8, 10, CV1800_DIV_FLAG, + 0); +static CV1800_GATE(clk_axi4_sd1, clk_axi4_bus_parents, + REG_CLK_EN_0, 21, + 0); +static CV1800_BYPASS_MUX(clk_sd1, clk_axi4_parents, + REG_CLK_EN_0, 22, + REG_DIV_CLK_SD1, 16, 5, 15, CV1800_DIV_FLAG, + REG_DIV_CLK_SD1, 8, 2, + REG_CLK_BYP_0, 7, + 0); +static CV1800_DIV(clk_sd1_100k, clk_parents_1m, + REG_CLK_EN_0, 23, + REG_DIV_CLK_SD1_100K, 16, 8, 10, CV1800_DIV_FLAG, + 0); + +/* SPI NAND */ +static CV1800_BYPASS_MUX(clk_spi_nand, clk_axi4_parents, + REG_CLK_EN_0, 24, + REG_DIV_CLK_SPI_NAND, 16, 5, 8, CV1800_DIV_FLAG, + REG_DIV_CLK_SPI_NAND, 8, 2, + REG_CLK_BYP_0, 8, + 0); + +/* GPIO */ +static CV1800_DIV(clk_gpio_db, clk_parents_1m, + REG_CLK_EN_0, 31, + REG_DIV_CLK_GPIO_DB, 16, 16, 10, CV1800_DIV_FLAG, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_apb_gpio, clk_axi6_bus_parents, + REG_CLK_EN_0, 29, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_apb_gpio_intr, clk_axi6_bus_parents, + REG_CLK_EN_0, 30, + CLK_IS_CRITICAL); + +/* ETH */ +static CV1800_BYPASS_DIV(clk_eth0_500m, clk_bypass_fpll_parents, + REG_CLK_EN_0, 25, + REG_DIV_CLK_GPIO_DB, 16, 4, 3, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 9, + 0); +static CV1800_GATE(clk_axi4_eth0, clk_axi4_bus_parents, + REG_CLK_EN_0, 26, + 0); +static CV1800_BYPASS_DIV(clk_eth1_500m, clk_bypass_fpll_parents, + REG_CLK_EN_0, 27, + REG_DIV_CLK_GPIO_DB, 16, 4, 3, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 10, + 0); +static CV1800_GATE(clk_axi4_eth1, clk_axi4_bus_parents, + REG_CLK_EN_0, 28, + 0); + +/* SF */ +static CV1800_GATE(clk_ahb_sf, clk_axi4_bus_parents, + REG_CLK_EN_1, 0, + 0); +static CV1800_GATE(clk_ahb_sf1, clk_axi4_bus_parents, + REG_CLK_EN_3, 27, + 0); + +/* AUDSRC */ +static CV1800_ACLK(clk_a24m, clk_mipimpll_parents, + REG_APLL_FRAC_DIV_CTRL, 0, + REG_APLL_FRAC_DIV_CTRL, 3, + REG_APLL_FRAC_DIV_CTRL, 1, + REG_APLL_FRAC_DIV_CTRL, 2, + REG_APLL_FRAC_DIV_M, 0, 22, CV1800_DIV_FLAG, + REG_APLL_FRAC_DIV_N, 0, 22, CV1800_DIV_FLAG, + 24576000, + 0); + +static const struct clk_parent_data clk_aud_parents[] = { + { .index = 0 }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_a24m.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_audsrc, clk_aud_parents, + REG_CLK_EN_4, 1, + REG_DIV_CLK_AUDSRC, 16, 8, 18, CV1800_DIV_FLAG, + REG_DIV_CLK_AUDSRC, 8, 2, + REG_CLK_BYP_1, 2, + 0); +static CV1800_GATE(clk_apb_audsrc, clk_axi4_bus_parents, + REG_CLK_EN_4, 2, + 0); + +/* SDMA */ +static CV1800_GATE(clk_sdma_axi, clk_axi4_bus_parents, + REG_CLK_EN_1, 1, + 0); +static CV1800_BYPASS_MUX(clk_sdma_aud0, clk_aud_parents, + REG_CLK_EN_1, 2, + REG_DIV_CLK_SDMA_AUD0, 16, 8, 18, CV1800_DIV_FLAG, + REG_DIV_CLK_SDMA_AUD0, 8, 2, + REG_CLK_BYP_0, 11, + 0); +static CV1800_BYPASS_MUX(clk_sdma_aud1, clk_aud_parents, + REG_CLK_EN_1, 3, + REG_DIV_CLK_SDMA_AUD1, 16, 8, 18, CV1800_DIV_FLAG, + REG_DIV_CLK_SDMA_AUD1, 8, 2, + REG_CLK_BYP_0, 12, + 0); +static CV1800_BYPASS_MUX(clk_sdma_aud2, clk_aud_parents, + REG_CLK_EN_1, 3, + REG_DIV_CLK_SDMA_AUD2, 16, 8, 18, CV1800_DIV_FLAG, + REG_DIV_CLK_SDMA_AUD2, 8, 2, + REG_CLK_BYP_0, 13, + 0); +static CV1800_BYPASS_MUX(clk_sdma_aud3, clk_aud_parents, + REG_CLK_EN_1, 3, + REG_DIV_CLK_SDMA_AUD3, 16, 8, 18, CV1800_DIV_FLAG, + REG_DIV_CLK_SDMA_AUD3, 8, 2, + REG_CLK_BYP_0, 14, + 0); + +/* SPI */ +static CV1800_GATE(clk_apb_spi0, clk_axi4_bus_parents, + REG_CLK_EN_1, 9, + 0); +static CV1800_GATE(clk_apb_spi1, clk_axi4_bus_parents, + REG_CLK_EN_1, 10, + 0); +static CV1800_GATE(clk_apb_spi2, clk_axi4_bus_parents, + REG_CLK_EN_1, 11, + 0); +static CV1800_GATE(clk_apb_spi3, clk_axi4_bus_parents, + REG_CLK_EN_1, 12, + 0); +static CV1800_BYPASS_DIV(clk_spi, clk_bypass_fpll_parents, + REG_CLK_EN_3, 6, + REG_DIV_CLK_SPI, 16, 6, 8, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 30, + 0); + +/* UART */ +static CV1800_GATE(clk_uart0, clk_uart_parents, + REG_CLK_EN_1, 14, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_apb_uart0, clk_axi4_bus_parents, + REG_CLK_EN_1, 15, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_uart1, clk_uart_parents, + REG_CLK_EN_1, 16, + 0); +static CV1800_GATE(clk_apb_uart1, clk_axi4_bus_parents, + REG_CLK_EN_1, 17, + 0); +static CV1800_GATE(clk_uart2, clk_uart_parents, + REG_CLK_EN_1, 18, + 0); +static CV1800_GATE(clk_apb_uart2, clk_axi4_bus_parents, + REG_CLK_EN_1, 19, + 0); +static CV1800_GATE(clk_uart3, clk_uart_parents, + REG_CLK_EN_1, 20, + 0); +static CV1800_GATE(clk_apb_uart3, clk_axi4_bus_parents, + REG_CLK_EN_1, 21, + 0); +static CV1800_GATE(clk_uart4, clk_uart_parents, + REG_CLK_EN_1, 22, + 0); +static CV1800_GATE(clk_apb_uart4, clk_axi4_bus_parents, + REG_CLK_EN_1, 23, + 0); + +/* I2S */ +static CV1800_GATE(clk_apb_i2s0, clk_axi4_bus_parents, + REG_CLK_EN_1, 24, + 0); +static CV1800_GATE(clk_apb_i2s1, clk_axi4_bus_parents, + REG_CLK_EN_1, 25, + 0); +static CV1800_GATE(clk_apb_i2s2, clk_axi4_bus_parents, + REG_CLK_EN_1, 26, + 0); +static CV1800_GATE(clk_apb_i2s3, clk_axi4_bus_parents, + REG_CLK_EN_1, 27, + 0); + +/* DEBUG */ +static CV1800_GATE(clk_debug, osc_parents, + REG_CLK_EN_0, 13, + CLK_IS_CRITICAL); +static CV1800_BYPASS_DIV(clk_ap_debug, clk_bypass_fpll_parents, + REG_CLK_EN_4, 5, + REG_DIV_CLK_AP_DEBUG, 16, 4, 5, CV1800_DIV_FLAG, + REG_CLK_BYP_1, 4, + CLK_IS_CRITICAL); + +/* DDR */ +static CV1800_GATE(clk_ddr_axi_reg, clk_axi6_bus_parents, + REG_CLK_EN_0, 7, + CLK_IS_CRITICAL); + +/* I2C */ +static CV1800_GATE(clk_apb_i2c, clk_axi4_bus_parents, + REG_CLK_EN_1, 6, + 0); +static CV1800_BYPASS_DIV(clk_i2c, clk_bypass_axi6_bus_parents, + REG_CLK_EN_3, 7, + REG_DIV_CLK_I2C, 16, 4, 1, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 31, + 0); +static CV1800_GATE(clk_apb_i2c0, clk_axi4_bus_parents, + REG_CLK_EN_3, 17, + 0); +static CV1800_GATE(clk_apb_i2c1, clk_axi4_bus_parents, + REG_CLK_EN_3, 18, + 0); +static CV1800_GATE(clk_apb_i2c2, clk_axi4_bus_parents, + REG_CLK_EN_3, 19, + 0); +static CV1800_GATE(clk_apb_i2c3, clk_axi4_bus_parents, + REG_CLK_EN_3, 20, + 0); +static CV1800_GATE(clk_apb_i2c4, clk_axi4_bus_parents, + REG_CLK_EN_3, 21, + 0); + +/* USB */ +static CV1800_GATE(clk_axi4_usb, clk_axi4_bus_parents, + REG_CLK_EN_1, 28, + 0); +static CV1800_GATE(clk_apb_usb, clk_axi4_bus_parents, + REG_CLK_EN_1, 29, + 0); +static CV1800_BYPASS_FIXED_DIV(clk_usb_125m, clk_bypass_fpll_parents, + REG_CLK_EN_1, 30, + 12, + REG_CLK_BYP_0, 17, + CLK_SET_RATE_PARENT); +static CV1800_FIXED_DIV(clk_usb_33k, clk_parents_1m, + REG_CLK_EN_1, 31, + 3, + 0); +static CV1800_BYPASS_FIXED_DIV(clk_usb_12m, clk_bypass_fpll_parents, + REG_CLK_EN_2, 0, + 125, + REG_CLK_BYP_0, 18, + CLK_SET_RATE_PARENT); + +/* VIP SYS */ +static const struct clk_parent_data clk_vip_sys_parents[] = { + { .index = 0 }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_cam0pll.common.hw }, + { .hw = &clk_disppll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; +static const struct clk_parent_data clk_disp_vip_parents[] = { + { .index = 0 }, + { .hw = &clk_disppll.common.hw }, +}; + +static CV1800_BYPASS_DIV(clk_dsi_esc, clk_bypass_axi6_bus_parents, + REG_CLK_EN_2, 3, + REG_DIV_CLK_DSI_ESC, 16, 4, 5, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 21, + 0); +static CV1800_BYPASS_MUX(clk_axi_vip, clk_vip_sys_parents, + REG_CLK_EN_2, 4, + REG_DIV_CLK_AXI_VIP, 16, 4, 3, CV1800_DIV_FLAG, + REG_DIV_CLK_AXI_VIP, 8, 2, + REG_CLK_BYP_0, 22, + 0); + +static const struct clk_parent_data clk_axi_vip_bus_parents[] = { + { .hw = &clk_axi_vip.mux.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_src_vip_sys_0, clk_vip_sys_parents, + REG_CLK_EN_2, 5, + REG_DIV_CLK_SRC_VIP_SYS_0, 16, 4, 6, CV1800_DIV_FLAG, + REG_DIV_CLK_SRC_VIP_SYS_0, 8, 2, + REG_CLK_BYP_0, 23, + 0); +static CV1800_BYPASS_MUX(clk_src_vip_sys_1, clk_vip_sys_parents, + REG_CLK_EN_2, 6, + REG_DIV_CLK_SRC_VIP_SYS_1, 16, 4, 6, CV1800_DIV_FLAG, + REG_DIV_CLK_SRC_VIP_SYS_1, 8, 2, + REG_CLK_BYP_0, 24, + 0); +static CV1800_BYPASS_DIV(clk_disp_src_vip, clk_disp_vip_parents, + REG_CLK_EN_2, 7, + REG_DIV_CLK_DISP_SRC_VIP, 16, 4, 8, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 25, + 0); +static CV1800_BYPASS_MUX(clk_src_vip_sys_2, clk_vip_sys_parents, + REG_CLK_EN_3, 29, + REG_DIV_CLK_SRC_VIP_SYS_2, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_SRC_VIP_SYS_2, 8, 2, + REG_CLK_BYP_1, 1, + 0); +static CV1800_GATE(clk_csi_mac0_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 18, + 0); +static CV1800_GATE(clk_csi_mac1_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 19, + 0); +static CV1800_GATE(clk_isp_top_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 20, + 0); +static CV1800_GATE(clk_img_d_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 21, + 0); +static CV1800_GATE(clk_img_v_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 22, + 0); +static CV1800_GATE(clk_sc_top_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 23, + 0); +static CV1800_GATE(clk_sc_d_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 24, + 0); +static CV1800_GATE(clk_sc_v1_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 25, + 0); +static CV1800_GATE(clk_sc_v2_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 26, + 0); +static CV1800_GATE(clk_sc_v3_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 27, + 0); +static CV1800_GATE(clk_dwa_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 28, + 0); +static CV1800_GATE(clk_bt_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 29, + 0); +static CV1800_GATE(clk_disp_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 30, + 0); +static CV1800_GATE(clk_dsi_mac_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 31, + 0); +static CV1800_GATE(clk_lvds0_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 0, + 0); +static CV1800_GATE(clk_lvds1_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 1, + 0); +static CV1800_GATE(clk_csi0_rx_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 2, + 0); +static CV1800_GATE(clk_csi1_rx_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 3, + 0); +static CV1800_GATE(clk_pad_vi_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 4, + 0); +static CV1800_GATE(clk_pad_vi1_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 30, + 0); +static CV1800_GATE(clk_cfg_reg_vip, clk_axi6_bus_parents, + REG_CLK_EN_3, 31, + 0); +static CV1800_GATE(clk_pad_vi2_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 7, + 0); +static CV1800_GATE(clk_csi_be_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 8, + 0); +static CV1800_GATE(clk_vip_ip0, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 9, + 0); +static CV1800_GATE(clk_vip_ip1, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 10, + 0); +static CV1800_GATE(clk_vip_ip2, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 11, + 0); +static CV1800_GATE(clk_vip_ip3, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 12, + 0); +static CV1800_BYPASS_MUX(clk_src_vip_sys_3, clk_vip_sys_parents, + REG_CLK_EN_4, 15, + REG_DIV_CLK_SRC_VIP_SYS_3, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_SRC_VIP_SYS_3, 8, 2, + REG_CLK_BYP_1, 8, + 0); +static CV1800_BYPASS_MUX(clk_src_vip_sys_4, clk_vip_sys_parents, + REG_CLK_EN_4, 16, + REG_DIV_CLK_SRC_VIP_SYS_4, 16, 4, 3, CV1800_DIV_FLAG, + REG_DIV_CLK_SRC_VIP_SYS_4, 8, 2, + REG_CLK_BYP_1, 9, + 0); +static CV1800_GATE(clk_ive_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 17, + 0); +static CV1800_GATE(clk_raw_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 18, + 0); +static CV1800_GATE(clk_osdc_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 19, + 0); +static CV1800_GATE(clk_csi_mac2_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 20, + 0); +static CV1800_GATE(clk_cam0_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 21, + 0); + +/* CAM OUT */ +static const struct clk_parent_data clk_cam_parents[] = { + { .hw = &clk_cam0pll.common.hw }, + { .hw = &clk_cam0pll_d2.common.hw }, + { .hw = &clk_cam0pll_d3.common.hw }, + { .hw = &clk_mipimpll_d3.common.hw }, +}; + +static CV1800_MUX(clk_cam0, clk_cam_parents, + REG_CLK_EN_2, 16, + REG_CLK_CAM0_SRC_DIV, 16, 6, 0, CV1800_DIV_FLAG, + REG_CLK_CAM0_SRC_DIV, 8, 2, + CLK_IGNORE_UNUSED); +static CV1800_MUX(clk_cam1, clk_cam_parents, + REG_CLK_EN_2, 17, + REG_CLK_CAM1_SRC_DIV, 16, 6, 0, CV1800_DIV_FLAG, + REG_CLK_CAM1_SRC_DIV, 8, 2, + CLK_IGNORE_UNUSED); + +/* VIDEO SUBSYS */ +static const struct clk_parent_data clk_axi_video_codec_parents[] = { + { .index = 0 }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_cam1pll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; +static const struct clk_parent_data clk_vc_src0_parents[] = { + { .index = 0 }, + { .hw = &clk_disppll.common.hw }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_cam1pll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; +static const struct clk_parent_data clk_vc_src1_parents[] = { + { .index = 0 }, + { .hw = &clk_cam1pll.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_axi_video_codec, clk_axi_video_codec_parents, + REG_CLK_EN_2, 8, + REG_DIV_CLK_AXI_VIDEO_CODEC, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_AXI_VIDEO_CODEC, 8, 2, + REG_CLK_BYP_0, 26, + 0); + +static const struct clk_parent_data clk_axi_video_codec_bus_parents[] = { + { .hw = &clk_axi_video_codec.mux.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_vc_src0, clk_vc_src0_parents, + REG_CLK_EN_2, 9, + REG_DIV_CLK_VC_SRC0, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_VC_SRC0, 8, 2, + REG_CLK_BYP_0, 27, + 0); + +static CV1800_GATE(clk_h264c, clk_axi_video_codec_bus_parents, + REG_CLK_EN_2, 10, + 0); +static CV1800_GATE(clk_h265c, clk_axi_video_codec_bus_parents, + REG_CLK_EN_2, 11, + 0); +static CV1800_GATE(clk_jpeg, clk_axi_video_codec_bus_parents, + REG_CLK_EN_2, 12, + CLK_IGNORE_UNUSED); +static CV1800_GATE(clk_apb_jpeg, clk_axi6_bus_parents, + REG_CLK_EN_2, 13, + CLK_IGNORE_UNUSED); +static CV1800_GATE(clk_apb_h264c, clk_axi6_bus_parents, + REG_CLK_EN_2, 14, + 0); +static CV1800_GATE(clk_apb_h265c, clk_axi6_bus_parents, + REG_CLK_EN_2, 15, + 0); +static CV1800_BYPASS_FIXED_DIV(clk_vc_src1, clk_vc_src1_parents, + REG_CLK_EN_3, 28, + 2, + REG_CLK_BYP_1, 0, + CLK_SET_RATE_PARENT); +static CV1800_BYPASS_FIXED_DIV(clk_vc_src2, clk_bypass_fpll_parents, + REG_CLK_EN_4, 3, + 3, + REG_CLK_BYP_1, 3, + CLK_SET_RATE_PARENT); + +/* VC SYS */ +static CV1800_GATE(clk_cfg_reg_vc, clk_axi6_bus_parents, + REG_CLK_EN_4, 0, + CLK_IGNORE_UNUSED); + +/* PWM */ +static CV1800_BYPASS_MUX(clk_pwm_src, clk_axi4_parents, + REG_CLK_EN_4, 4, + REG_DIV_CLK_PWM_SRC_0, 16, 6, 10, CV1800_DIV_FLAG, + REG_DIV_CLK_PWM_SRC_0, 8, 2, + REG_CLK_BYP_0, 15, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_pwm_parents[] = { + { .hw = &clk_pwm_src.mux.common.hw }, +}; + +static CV1800_GATE(clk_pwm, clk_pwm_parents, + REG_CLK_EN_1, 8, + CLK_IS_CRITICAL); + +/* C906 */ +static const struct clk_parent_data clk_c906_0_parents[] = { + { .index = 0 }, + { .hw = &clk_tpll.common.hw }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_mpll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; +static const struct clk_parent_data clk_c906_1_parents[] = { + { .index = 0 }, + { .hw = &clk_tpll.common.hw }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_disppll.common.hw }, + { .hw = &clk_mpll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; + +static const s8 clk_c906_parent2sel[] = { + -1, /* osc */ + 0, /* mux 0: clk_tpll(c906_0), clk_tpll(c906_1) */ + 0, /* mux 0: clk_a0pll(c906_0), clk_a0pll(c906_1) */ + 0, /* mux 0: clk_mipimpll(c906_0), clk_disppll(c906_1) */ + 0, /* mux 0: clk_mpll(c906_0), clk_mpll(c906_1) */ + 1 /* mux 1: clk_fpll(c906_0), clk_fpll(c906_1) */ +}; + +static const u8 clk_c906_sel2parent[2][4] = { + [0] = { + 1, + 2, + 3, + 4 + }, + [1] = { + 5, + 5, + 5, + 5 + }, +}; + +static CV1800_MMUX(clk_c906_0, clk_c906_0_parents, + REG_CLK_EN_4, 13, + REG_DIV_CLK_C906_0_0, 16, 4, 1, CV1800_DIV_FLAG, + REG_DIV_CLK_C906_0_1, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_C906_0_0, 8, 2, + REG_DIV_CLK_C906_0_1, 8, 2, + REG_CLK_BYP_1, 6, + REG_CLK_SEL_0, 23, + clk_c906_parent2sel, + clk_c906_sel2parent[0], clk_c906_sel2parent[1], + CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE); +static CV1800_MMUX(clk_c906_1, clk_c906_1_parents, + REG_CLK_EN_4, 14, + REG_DIV_CLK_C906_1_0, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_C906_1_1, 16, 4, 3, CV1800_DIV_FLAG, + REG_DIV_CLK_C906_1_0, 8, 2, + REG_DIV_CLK_C906_1_1, 8, 2, + REG_CLK_BYP_1, 7, + REG_CLK_SEL_0, 24, + clk_c906_parent2sel, + clk_c906_sel2parent[0], clk_c906_sel2parent[1], + CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE); + +/* A53 */ +static CV1800_BYPASS_DIV(clk_cpu_axi0, clk_axi4_parents, + REG_CLK_EN_0, 1, + REG_DIV_CLK_CPU_AXI0, 16, 4, 3, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 1, + CLK_IS_CRITICAL); +static CV1800_BYPASS_DIV(clk_cpu_gic, clk_bypass_fpll_parents, + REG_CLK_EN_0, 2, + REG_DIV_CLK_CPU_GIC, 16, 4, 5, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 2, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_xtal_ap, osc_parents, + REG_CLK_EN_0, 3, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_a53_parents[] = { + { .index = 0 }, + { .hw = &clk_tpll.common.hw }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_mpll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; + +static const s8 clk_a53_parent2sel[] = { + -1, /* osc */ + 0, /* mux 0: clk_tpll */ + 0, /* mux 0: clk_a0pll */ + 0, /* mux 0: clk_mipimpll */ + 0, /* mux 0: clk_mpll */ + 1 /* mux 1: clk_fpll */ +}; + +static const u8 clk_a53_sel2parent[2][4] = { + [0] = { + 1, + 2, + 3, + 4 + }, + [1] = { + 5, + 5, + 5, + 5 + }, +}; + +/* + * Clock for A53 cpu in the CV18XX/SG200X series. + * For CV180X and CV181X series, this clock is not used, but can not + * be set to bypass mode, or the SoC will hang. + */ +static CV1800_MMUX(clk_a53, clk_a53_parents, + REG_CLK_EN_0, 0, + REG_DIV_CLK_A53_0, 16, 4, 1, CV1800_DIV_FLAG, + REG_DIV_CLK_A53_1, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_A53_0, 8, 2, + REG_DIV_CLK_A53_1, 8, 2, + REG_CLK_BYP_0, 0, + REG_CLK_SEL_0, 0, + clk_a53_parent2sel, + clk_a53_sel2parent[0], clk_a53_sel2parent[1], + CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE); + +static struct clk_hw_onecell_data cv1800_hw_clks = { + .num = CV1800_CLK_MAX, + .hws = { + [CLK_MPLL] = &clk_mpll.common.hw, + [CLK_TPLL] = &clk_tpll.common.hw, + [CLK_FPLL] = &clk_fpll.common.hw, + [CLK_MIPIMPLL] = &clk_mipimpll.common.hw, + [CLK_A0PLL] = &clk_a0pll.common.hw, + [CLK_DISPPLL] = &clk_disppll.common.hw, + [CLK_CAM0PLL] = &clk_cam0pll.common.hw, + [CLK_CAM1PLL] = &clk_cam1pll.common.hw, + + [CLK_MIPIMPLL_D3] = &clk_mipimpll_d3.common.hw, + [CLK_CAM0PLL_D2] = &clk_cam0pll_d2.common.hw, + [CLK_CAM0PLL_D3] = &clk_cam0pll_d3.common.hw, + + [CLK_TPU] = &clk_tpu.mux.common.hw, + [CLK_TPU_FAB] = &clk_tpu_fab.common.hw, + [CLK_AHB_ROM] = &clk_ahb_rom.common.hw, + [CLK_DDR_AXI_REG] = &clk_ddr_axi_reg.common.hw, + [CLK_RTC_25M] = &clk_rtc_25m.common.hw, + [CLK_SRC_RTC_SYS_0] = &clk_src_rtc_sys_0.div.common.hw, + [CLK_TEMPSEN] = &clk_tempsen.common.hw, + [CLK_SARADC] = &clk_saradc.common.hw, + [CLK_EFUSE] = &clk_efuse.common.hw, + [CLK_APB_EFUSE] = &clk_apb_efuse.common.hw, + [CLK_DEBUG] = &clk_debug.common.hw, + [CLK_AP_DEBUG] = &clk_ap_debug.div.common.hw, + [CLK_XTAL_MISC] = &clk_xtal_misc.common.hw, + [CLK_AXI4_EMMC] = &clk_axi4_emmc.common.hw, + [CLK_EMMC] = &clk_emmc.mux.common.hw, + [CLK_EMMC_100K] = &clk_emmc_100k.common.hw, + [CLK_AXI4_SD0] = &clk_axi4_sd0.common.hw, + [CLK_SD0] = &clk_sd0.mux.common.hw, + [CLK_SD0_100K] = &clk_sd0_100k.common.hw, + [CLK_AXI4_SD1] = &clk_axi4_sd1.common.hw, + [CLK_SD1] = &clk_sd1.mux.common.hw, + [CLK_SD1_100K] = &clk_sd1_100k.common.hw, + [CLK_SPI_NAND] = &clk_spi_nand.mux.common.hw, + [CLK_ETH0_500M] = &clk_eth0_500m.div.common.hw, + [CLK_AXI4_ETH0] = &clk_axi4_eth0.common.hw, + [CLK_ETH1_500M] = &clk_eth1_500m.div.common.hw, + [CLK_AXI4_ETH1] = &clk_axi4_eth1.common.hw, + [CLK_APB_GPIO] = &clk_apb_gpio.common.hw, + [CLK_APB_GPIO_INTR] = &clk_apb_gpio_intr.common.hw, + [CLK_GPIO_DB] = &clk_gpio_db.common.hw, + [CLK_AHB_SF] = &clk_ahb_sf.common.hw, + [CLK_AHB_SF1] = &clk_ahb_sf1.common.hw, + [CLK_A24M] = &clk_a24m.common.hw, + [CLK_AUDSRC] = &clk_audsrc.mux.common.hw, + [CLK_APB_AUDSRC] = &clk_apb_audsrc.common.hw, + [CLK_SDMA_AXI] = &clk_sdma_axi.common.hw, + [CLK_SDMA_AUD0] = &clk_sdma_aud0.mux.common.hw, + [CLK_SDMA_AUD1] = &clk_sdma_aud1.mux.common.hw, + [CLK_SDMA_AUD2] = &clk_sdma_aud2.mux.common.hw, + [CLK_SDMA_AUD3] = &clk_sdma_aud3.mux.common.hw, + [CLK_I2C] = &clk_i2c.div.common.hw, + [CLK_APB_I2C] = &clk_apb_i2c.common.hw, + [CLK_APB_I2C0] = &clk_apb_i2c0.common.hw, + [CLK_APB_I2C1] = &clk_apb_i2c1.common.hw, + [CLK_APB_I2C2] = &clk_apb_i2c2.common.hw, + [CLK_APB_I2C3] = &clk_apb_i2c3.common.hw, + [CLK_APB_I2C4] = &clk_apb_i2c4.common.hw, + [CLK_APB_WDT] = &clk_apb_wdt.common.hw, + [CLK_PWM_SRC] = &clk_pwm_src.mux.common.hw, + [CLK_PWM] = &clk_pwm.common.hw, + [CLK_SPI] = &clk_spi.div.common.hw, + [CLK_APB_SPI0] = &clk_apb_spi0.common.hw, + [CLK_APB_SPI1] = &clk_apb_spi1.common.hw, + [CLK_APB_SPI2] = &clk_apb_spi2.common.hw, + [CLK_APB_SPI3] = &clk_apb_spi3.common.hw, + [CLK_1M] = &clk_1m.common.hw, + [CLK_CAM0_200] = &clk_cam0_200.mux.common.hw, + [CLK_PM] = &clk_pm.common.hw, + [CLK_TIMER0] = &clk_timer0.common.hw, + [CLK_TIMER1] = &clk_timer1.common.hw, + [CLK_TIMER2] = &clk_timer2.common.hw, + [CLK_TIMER3] = &clk_timer3.common.hw, + [CLK_TIMER4] = &clk_timer4.common.hw, + [CLK_TIMER5] = &clk_timer5.common.hw, + [CLK_TIMER6] = &clk_timer6.common.hw, + [CLK_TIMER7] = &clk_timer7.common.hw, + [CLK_UART0] = &clk_uart0.common.hw, + [CLK_APB_UART0] = &clk_apb_uart0.common.hw, + [CLK_UART1] = &clk_uart1.common.hw, + [CLK_APB_UART1] = &clk_apb_uart1.common.hw, + [CLK_UART2] = &clk_uart2.common.hw, + [CLK_APB_UART2] = &clk_apb_uart2.common.hw, + [CLK_UART3] = &clk_uart3.common.hw, + [CLK_APB_UART3] = &clk_apb_uart3.common.hw, + [CLK_UART4] = &clk_uart4.common.hw, + [CLK_APB_UART4] = &clk_apb_uart4.common.hw, + [CLK_APB_I2S0] = &clk_apb_i2s0.common.hw, + [CLK_APB_I2S1] = &clk_apb_i2s1.common.hw, + [CLK_APB_I2S2] = &clk_apb_i2s2.common.hw, + [CLK_APB_I2S3] = &clk_apb_i2s3.common.hw, + [CLK_AXI4_USB] = &clk_axi4_usb.common.hw, + [CLK_APB_USB] = &clk_apb_usb.common.hw, + [CLK_USB_125M] = &clk_usb_125m.div.common.hw, + [CLK_USB_33K] = &clk_usb_33k.common.hw, + [CLK_USB_12M] = &clk_usb_12m.div.common.hw, + [CLK_AXI4] = &clk_axi4.mux.common.hw, + [CLK_AXI6] = &clk_axi6.div.common.hw, + [CLK_DSI_ESC] = &clk_dsi_esc.div.common.hw, + [CLK_AXI_VIP] = &clk_axi_vip.mux.common.hw, + [CLK_SRC_VIP_SYS_0] = &clk_src_vip_sys_0.mux.common.hw, + [CLK_SRC_VIP_SYS_1] = &clk_src_vip_sys_1.mux.common.hw, + [CLK_SRC_VIP_SYS_2] = &clk_src_vip_sys_2.mux.common.hw, + [CLK_SRC_VIP_SYS_3] = &clk_src_vip_sys_3.mux.common.hw, + [CLK_SRC_VIP_SYS_4] = &clk_src_vip_sys_4.mux.common.hw, + [CLK_CSI_BE_VIP] = &clk_csi_be_vip.common.hw, + [CLK_CSI_MAC0_VIP] = &clk_csi_mac0_vip.common.hw, + [CLK_CSI_MAC1_VIP] = &clk_csi_mac1_vip.common.hw, + [CLK_CSI_MAC2_VIP] = &clk_csi_mac2_vip.common.hw, + [CLK_CSI0_RX_VIP] = &clk_csi0_rx_vip.common.hw, + [CLK_CSI1_RX_VIP] = &clk_csi1_rx_vip.common.hw, + [CLK_ISP_TOP_VIP] = &clk_isp_top_vip.common.hw, + [CLK_IMG_D_VIP] = &clk_img_d_vip.common.hw, + [CLK_IMG_V_VIP] = &clk_img_v_vip.common.hw, + [CLK_SC_TOP_VIP] = &clk_sc_top_vip.common.hw, + [CLK_SC_D_VIP] = &clk_sc_d_vip.common.hw, + [CLK_SC_V1_VIP] = &clk_sc_v1_vip.common.hw, + [CLK_SC_V2_VIP] = &clk_sc_v2_vip.common.hw, + [CLK_SC_V3_VIP] = &clk_sc_v3_vip.common.hw, + [CLK_DWA_VIP] = &clk_dwa_vip.common.hw, + [CLK_BT_VIP] = &clk_bt_vip.common.hw, + [CLK_DISP_VIP] = &clk_disp_vip.common.hw, + [CLK_DSI_MAC_VIP] = &clk_dsi_mac_vip.common.hw, + [CLK_LVDS0_VIP] = &clk_lvds0_vip.common.hw, + [CLK_LVDS1_VIP] = &clk_lvds1_vip.common.hw, + [CLK_PAD_VI_VIP] = &clk_pad_vi_vip.common.hw, + [CLK_PAD_VI1_VIP] = &clk_pad_vi1_vip.common.hw, + [CLK_PAD_VI2_VIP] = &clk_pad_vi2_vip.common.hw, + [CLK_CFG_REG_VIP] = &clk_cfg_reg_vip.common.hw, + [CLK_VIP_IP0] = &clk_vip_ip0.common.hw, + [CLK_VIP_IP1] = &clk_vip_ip1.common.hw, + [CLK_VIP_IP2] = &clk_vip_ip2.common.hw, + [CLK_VIP_IP3] = &clk_vip_ip3.common.hw, + [CLK_IVE_VIP] = &clk_ive_vip.common.hw, + [CLK_RAW_VIP] = &clk_raw_vip.common.hw, + [CLK_OSDC_VIP] = &clk_osdc_vip.common.hw, + [CLK_CAM0_VIP] = &clk_cam0_vip.common.hw, + [CLK_AXI_VIDEO_CODEC] = &clk_axi_video_codec.mux.common.hw, + [CLK_VC_SRC0] = &clk_vc_src0.mux.common.hw, + [CLK_VC_SRC1] = &clk_vc_src1.div.common.hw, + [CLK_VC_SRC2] = &clk_vc_src2.div.common.hw, + [CLK_H264C] = &clk_h264c.common.hw, + [CLK_APB_H264C] = &clk_apb_h264c.common.hw, + [CLK_H265C] = &clk_h265c.common.hw, + [CLK_APB_H265C] = &clk_apb_h265c.common.hw, + [CLK_JPEG] = &clk_jpeg.common.hw, + [CLK_APB_JPEG] = &clk_apb_jpeg.common.hw, + [CLK_CAM0] = &clk_cam0.common.hw, + [CLK_CAM1] = &clk_cam1.common.hw, + [CLK_WGN] = &clk_wgn.common.hw, + [CLK_WGN0] = &clk_wgn0.common.hw, + [CLK_WGN1] = &clk_wgn1.common.hw, + [CLK_WGN2] = &clk_wgn2.common.hw, + [CLK_KEYSCAN] = &clk_keyscan.common.hw, + [CLK_CFG_REG_VC] = &clk_cfg_reg_vc.common.hw, + [CLK_C906_0] = &clk_c906_0.common.hw, + [CLK_C906_1] = &clk_c906_1.common.hw, + [CLK_A53] = &clk_a53.common.hw, + [CLK_CPU_AXI0] = &clk_cpu_axi0.div.common.hw, + [CLK_CPU_GIC] = &clk_cpu_gic.div.common.hw, + [CLK_XTAL_AP] = &clk_xtal_ap.common.hw, + }, +}; + +static void cv18xx_clk_disable_auto_pd(void __iomem *base) +{ + static const u16 CV1800_PD_CLK[] = { + REG_MIPIMPLL_CLK_CSR, + REG_A0PLL_CLK_CSR, + REG_DISPPLL_CLK_CSR, + REG_CAM0PLL_CLK_CSR, + REG_CAM1PLL_CLK_CSR, + }; + + u32 val; + int i; + + /* disable auto power down */ + for (i = 0; i < ARRAY_SIZE(CV1800_PD_CLK); i++) { + u32 reg = CV1800_PD_CLK[i]; + + val = readl(base + reg); + val |= GENMASK(12, 9); + val &= ~BIT(8); + writel(val, base + reg); + } +} + +static void cv18xx_clk_disable_a53(void __iomem *base) +{ + u32 val = readl(base + REG_CLK_BYP_0); + + /* Set bypass clock for clk_a53 */ + val |= BIT(0); + + /* Set bypass clock for clk_cpu_axi0 */ + val |= BIT(1); + + /* Set bypass clock for clk_cpu_gic */ + val |= BIT(2); + + writel(val, base + REG_CLK_BYP_0); +} + +static int cv1800_pre_init(struct device *dev, void __iomem *base, + struct cv1800_clk_ctrl *ctrl, + const struct cv1800_clk_desc *desc) +{ + u32 val = readl(base + REG_CLK_EN_2); + + /* disable unsupported clk_disp_src_vip */ + val &= ~BIT(7); + + writel(val, base + REG_CLK_EN_2); + + cv18xx_clk_disable_a53(base); + cv18xx_clk_disable_auto_pd(base); + + return 0; +} + +static const struct cv1800_clk_desc cv1800_desc = { + .clks_data = &cv1800_hw_clks, + .pre_init = cv1800_pre_init, +}; + +static struct clk_hw_onecell_data cv1810_hw_clks = { + .num = CV1810_CLK_MAX, + .hws = { + [CLK_MPLL] = &clk_mpll.common.hw, + [CLK_TPLL] = &clk_tpll.common.hw, + [CLK_FPLL] = &clk_fpll.common.hw, + [CLK_MIPIMPLL] = &clk_mipimpll.common.hw, + [CLK_A0PLL] = &clk_a0pll.common.hw, + [CLK_DISPPLL] = &clk_disppll.common.hw, + [CLK_CAM0PLL] = &clk_cam0pll.common.hw, + [CLK_CAM1PLL] = &clk_cam1pll.common.hw, + + [CLK_MIPIMPLL_D3] = &clk_mipimpll_d3.common.hw, + [CLK_CAM0PLL_D2] = &clk_cam0pll_d2.common.hw, + [CLK_CAM0PLL_D3] = &clk_cam0pll_d3.common.hw, + + [CLK_TPU] = &clk_tpu.mux.common.hw, + [CLK_TPU_FAB] = &clk_tpu_fab.common.hw, + [CLK_AHB_ROM] = &clk_ahb_rom.common.hw, + [CLK_DDR_AXI_REG] = &clk_ddr_axi_reg.common.hw, + [CLK_RTC_25M] = &clk_rtc_25m.common.hw, + [CLK_SRC_RTC_SYS_0] = &clk_src_rtc_sys_0.div.common.hw, + [CLK_TEMPSEN] = &clk_tempsen.common.hw, + [CLK_SARADC] = &clk_saradc.common.hw, + [CLK_EFUSE] = &clk_efuse.common.hw, + [CLK_APB_EFUSE] = &clk_apb_efuse.common.hw, + [CLK_DEBUG] = &clk_debug.common.hw, + [CLK_AP_DEBUG] = &clk_ap_debug.div.common.hw, + [CLK_XTAL_MISC] = &clk_xtal_misc.common.hw, + [CLK_AXI4_EMMC] = &clk_axi4_emmc.common.hw, + [CLK_EMMC] = &clk_emmc.mux.common.hw, + [CLK_EMMC_100K] = &clk_emmc_100k.common.hw, + [CLK_AXI4_SD0] = &clk_axi4_sd0.common.hw, + [CLK_SD0] = &clk_sd0.mux.common.hw, + [CLK_SD0_100K] = &clk_sd0_100k.common.hw, + [CLK_AXI4_SD1] = &clk_axi4_sd1.common.hw, + [CLK_SD1] = &clk_sd1.mux.common.hw, + [CLK_SD1_100K] = &clk_sd1_100k.common.hw, + [CLK_SPI_NAND] = &clk_spi_nand.mux.common.hw, + [CLK_ETH0_500M] = &clk_eth0_500m.div.common.hw, + [CLK_AXI4_ETH0] = &clk_axi4_eth0.common.hw, + [CLK_ETH1_500M] = &clk_eth1_500m.div.common.hw, + [CLK_AXI4_ETH1] = &clk_axi4_eth1.common.hw, + [CLK_APB_GPIO] = &clk_apb_gpio.common.hw, + [CLK_APB_GPIO_INTR] = &clk_apb_gpio_intr.common.hw, + [CLK_GPIO_DB] = &clk_gpio_db.common.hw, + [CLK_AHB_SF] = &clk_ahb_sf.common.hw, + [CLK_AHB_SF1] = &clk_ahb_sf1.common.hw, + [CLK_A24M] = &clk_a24m.common.hw, + [CLK_AUDSRC] = &clk_audsrc.mux.common.hw, + [CLK_APB_AUDSRC] = &clk_apb_audsrc.common.hw, + [CLK_SDMA_AXI] = &clk_sdma_axi.common.hw, + [CLK_SDMA_AUD0] = &clk_sdma_aud0.mux.common.hw, + [CLK_SDMA_AUD1] = &clk_sdma_aud1.mux.common.hw, + [CLK_SDMA_AUD2] = &clk_sdma_aud2.mux.common.hw, + [CLK_SDMA_AUD3] = &clk_sdma_aud3.mux.common.hw, + [CLK_I2C] = &clk_i2c.div.common.hw, + [CLK_APB_I2C] = &clk_apb_i2c.common.hw, + [CLK_APB_I2C0] = &clk_apb_i2c0.common.hw, + [CLK_APB_I2C1] = &clk_apb_i2c1.common.hw, + [CLK_APB_I2C2] = &clk_apb_i2c2.common.hw, + [CLK_APB_I2C3] = &clk_apb_i2c3.common.hw, + [CLK_APB_I2C4] = &clk_apb_i2c4.common.hw, + [CLK_APB_WDT] = &clk_apb_wdt.common.hw, + [CLK_PWM_SRC] = &clk_pwm_src.mux.common.hw, + [CLK_PWM] = &clk_pwm.common.hw, + [CLK_SPI] = &clk_spi.div.common.hw, + [CLK_APB_SPI0] = &clk_apb_spi0.common.hw, + [CLK_APB_SPI1] = &clk_apb_spi1.common.hw, + [CLK_APB_SPI2] = &clk_apb_spi2.common.hw, + [CLK_APB_SPI3] = &clk_apb_spi3.common.hw, + [CLK_1M] = &clk_1m.common.hw, + [CLK_CAM0_200] = &clk_cam0_200.mux.common.hw, + [CLK_PM] = &clk_pm.common.hw, + [CLK_TIMER0] = &clk_timer0.common.hw, + [CLK_TIMER1] = &clk_timer1.common.hw, + [CLK_TIMER2] = &clk_timer2.common.hw, + [CLK_TIMER3] = &clk_timer3.common.hw, + [CLK_TIMER4] = &clk_timer4.common.hw, + [CLK_TIMER5] = &clk_timer5.common.hw, + [CLK_TIMER6] = &clk_timer6.common.hw, + [CLK_TIMER7] = &clk_timer7.common.hw, + [CLK_UART0] = &clk_uart0.common.hw, + [CLK_APB_UART0] = &clk_apb_uart0.common.hw, + [CLK_UART1] = &clk_uart1.common.hw, + [CLK_APB_UART1] = &clk_apb_uart1.common.hw, + [CLK_UART2] = &clk_uart2.common.hw, + [CLK_APB_UART2] = &clk_apb_uart2.common.hw, + [CLK_UART3] = &clk_uart3.common.hw, + [CLK_APB_UART3] = &clk_apb_uart3.common.hw, + [CLK_UART4] = &clk_uart4.common.hw, + [CLK_APB_UART4] = &clk_apb_uart4.common.hw, + [CLK_APB_I2S0] = &clk_apb_i2s0.common.hw, + [CLK_APB_I2S1] = &clk_apb_i2s1.common.hw, + [CLK_APB_I2S2] = &clk_apb_i2s2.common.hw, + [CLK_APB_I2S3] = &clk_apb_i2s3.common.hw, + [CLK_AXI4_USB] = &clk_axi4_usb.common.hw, + [CLK_APB_USB] = &clk_apb_usb.common.hw, + [CLK_USB_125M] = &clk_usb_125m.div.common.hw, + [CLK_USB_33K] = &clk_usb_33k.common.hw, + [CLK_USB_12M] = &clk_usb_12m.div.common.hw, + [CLK_AXI4] = &clk_axi4.mux.common.hw, + [CLK_AXI6] = &clk_axi6.div.common.hw, + [CLK_DSI_ESC] = &clk_dsi_esc.div.common.hw, + [CLK_AXI_VIP] = &clk_axi_vip.mux.common.hw, + [CLK_SRC_VIP_SYS_0] = &clk_src_vip_sys_0.mux.common.hw, + [CLK_SRC_VIP_SYS_1] = &clk_src_vip_sys_1.mux.common.hw, + [CLK_SRC_VIP_SYS_2] = &clk_src_vip_sys_2.mux.common.hw, + [CLK_SRC_VIP_SYS_3] = &clk_src_vip_sys_3.mux.common.hw, + [CLK_SRC_VIP_SYS_4] = &clk_src_vip_sys_4.mux.common.hw, + [CLK_CSI_BE_VIP] = &clk_csi_be_vip.common.hw, + [CLK_CSI_MAC0_VIP] = &clk_csi_mac0_vip.common.hw, + [CLK_CSI_MAC1_VIP] = &clk_csi_mac1_vip.common.hw, + [CLK_CSI_MAC2_VIP] = &clk_csi_mac2_vip.common.hw, + [CLK_CSI0_RX_VIP] = &clk_csi0_rx_vip.common.hw, + [CLK_CSI1_RX_VIP] = &clk_csi1_rx_vip.common.hw, + [CLK_ISP_TOP_VIP] = &clk_isp_top_vip.common.hw, + [CLK_IMG_D_VIP] = &clk_img_d_vip.common.hw, + [CLK_IMG_V_VIP] = &clk_img_v_vip.common.hw, + [CLK_SC_TOP_VIP] = &clk_sc_top_vip.common.hw, + [CLK_SC_D_VIP] = &clk_sc_d_vip.common.hw, + [CLK_SC_V1_VIP] = &clk_sc_v1_vip.common.hw, + [CLK_SC_V2_VIP] = &clk_sc_v2_vip.common.hw, + [CLK_SC_V3_VIP] = &clk_sc_v3_vip.common.hw, + [CLK_DWA_VIP] = &clk_dwa_vip.common.hw, + [CLK_BT_VIP] = &clk_bt_vip.common.hw, + [CLK_DISP_VIP] = &clk_disp_vip.common.hw, + [CLK_DSI_MAC_VIP] = &clk_dsi_mac_vip.common.hw, + [CLK_LVDS0_VIP] = &clk_lvds0_vip.common.hw, + [CLK_LVDS1_VIP] = &clk_lvds1_vip.common.hw, + [CLK_PAD_VI_VIP] = &clk_pad_vi_vip.common.hw, + [CLK_PAD_VI1_VIP] = &clk_pad_vi1_vip.common.hw, + [CLK_PAD_VI2_VIP] = &clk_pad_vi2_vip.common.hw, + [CLK_CFG_REG_VIP] = &clk_cfg_reg_vip.common.hw, + [CLK_VIP_IP0] = &clk_vip_ip0.common.hw, + [CLK_VIP_IP1] = &clk_vip_ip1.common.hw, + [CLK_VIP_IP2] = &clk_vip_ip2.common.hw, + [CLK_VIP_IP3] = &clk_vip_ip3.common.hw, + [CLK_IVE_VIP] = &clk_ive_vip.common.hw, + [CLK_RAW_VIP] = &clk_raw_vip.common.hw, + [CLK_OSDC_VIP] = &clk_osdc_vip.common.hw, + [CLK_CAM0_VIP] = &clk_cam0_vip.common.hw, + [CLK_AXI_VIDEO_CODEC] = &clk_axi_video_codec.mux.common.hw, + [CLK_VC_SRC0] = &clk_vc_src0.mux.common.hw, + [CLK_VC_SRC1] = &clk_vc_src1.div.common.hw, + [CLK_VC_SRC2] = &clk_vc_src2.div.common.hw, + [CLK_H264C] = &clk_h264c.common.hw, + [CLK_APB_H264C] = &clk_apb_h264c.common.hw, + [CLK_H265C] = &clk_h265c.common.hw, + [CLK_APB_H265C] = &clk_apb_h265c.common.hw, + [CLK_JPEG] = &clk_jpeg.common.hw, + [CLK_APB_JPEG] = &clk_apb_jpeg.common.hw, + [CLK_CAM0] = &clk_cam0.common.hw, + [CLK_CAM1] = &clk_cam1.common.hw, + [CLK_WGN] = &clk_wgn.common.hw, + [CLK_WGN0] = &clk_wgn0.common.hw, + [CLK_WGN1] = &clk_wgn1.common.hw, + [CLK_WGN2] = &clk_wgn2.common.hw, + [CLK_KEYSCAN] = &clk_keyscan.common.hw, + [CLK_CFG_REG_VC] = &clk_cfg_reg_vc.common.hw, + [CLK_C906_0] = &clk_c906_0.common.hw, + [CLK_C906_1] = &clk_c906_1.common.hw, + [CLK_A53] = &clk_a53.common.hw, + [CLK_CPU_AXI0] = &clk_cpu_axi0.div.common.hw, + [CLK_CPU_GIC] = &clk_cpu_gic.div.common.hw, + [CLK_XTAL_AP] = &clk_xtal_ap.common.hw, + [CLK_DISP_SRC_VIP] = &clk_disp_src_vip.div.common.hw, + }, +}; + +static int cv1810_pre_init(struct device *dev, void __iomem *base, + struct cv1800_clk_ctrl *ctrl, + const struct cv1800_clk_desc *desc) +{ + cv18xx_clk_disable_a53(base); + cv18xx_clk_disable_auto_pd(base); + + return 0; +} + +static const struct cv1800_clk_desc cv1810_desc = { + .clks_data = &cv1810_hw_clks, + .pre_init = cv1810_pre_init, +}; + +static int sg2000_pre_init(struct device *dev, void __iomem *base, + struct cv1800_clk_ctrl *ctrl, + const struct cv1800_clk_desc *desc) +{ + cv18xx_clk_disable_auto_pd(base); + + return 0; +} + +static const struct cv1800_clk_desc sg2000_desc = { + .clks_data = &cv1810_hw_clks, + .pre_init = sg2000_pre_init, +}; + +static int cv1800_clk_init_ctrl(struct device *dev, void __iomem *reg, + struct cv1800_clk_ctrl *ctrl, + const struct cv1800_clk_desc *desc) +{ + int i, ret; + + ctrl->desc = desc; + spin_lock_init(&ctrl->lock); + + for (i = 0; i < desc->clks_data->num; i++) { + struct clk_hw *hw = desc->clks_data->hws[i]; + struct cv1800_clk_common *common; + const char *name; + + if (!hw) + continue; + + name = hw->init->name; + + common = hw_to_cv1800_clk_common(hw); + common->base = reg; + common->lock = &ctrl->lock; + + ret = devm_clk_hw_register(dev, hw); + if (ret) { + dev_err(dev, "Couldn't register clock %d - %s\n", + i, name); + return ret; + } + } + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, + desc->clks_data); +} + +static int cv1800_clk_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + void __iomem *reg; + int ret; + const struct cv1800_clk_desc *desc; + struct cv1800_clk_ctrl *ctrl; + + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + desc = device_get_match_data(dev); + if (!desc) { + dev_err(dev, "no match data for platform\n"); + return -EINVAL; + } + + ctrl = devm_kzalloc(dev, sizeof(*ctrl), GFP_KERNEL); + if (!ctrl) + return -ENOMEM; + + if (desc->pre_init) { + ret = desc->pre_init(dev, reg, ctrl, desc); + if (ret) + return ret; + } + + return cv1800_clk_init_ctrl(dev, reg, ctrl, desc); +} + +static const struct of_device_id cv1800_clk_ids[] = { + { .compatible = "sophgo,cv1800-clk", .data = &cv1800_desc }, + { .compatible = "sophgo,cv1810-clk", .data = &cv1810_desc }, + { .compatible = "sophgo,sg2000-clk", .data = &sg2000_desc }, + { } +}; +MODULE_DEVICE_TABLE(of, cv1800_clk_ids); + +static struct platform_driver cv1800_clk_driver = { + .probe = cv1800_clk_probe, + .driver = { + .name = "cv1800-clk", + .suppress_bind_attrs = true, + .of_match_table = cv1800_clk_ids, + }, +}; +module_platform_driver(cv1800_clk_driver); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sophgo/clk-cv1800.h b/drivers/clk/sophgo/clk-cv1800.h new file mode 100644 index 000000000000..1e7107b5d05e --- /dev/null +++ b/drivers/clk/sophgo/clk-cv1800.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023 Inochi Amaoto <inochiama@outlook.com> + */ + +#ifndef _CLK_SOPHGO_CV1800_H_ +#define _CLK_SOPHGO_CV1800_H_ + +#include <dt-bindings/clock/sophgo,cv1800.h> + +#define CV1800_CLK_MAX (CLK_XTAL_AP + 1) +#define CV1810_CLK_MAX (CLK_DISP_SRC_VIP + 1) + +#define REG_PLL_G2_CTRL 0x800 +#define REG_PLL_G2_STATUS 0x804 +#define REG_MIPIMPLL_CSR 0x808 +#define REG_A0PLL_CSR 0x80C +#define REG_DISPPLL_CSR 0x810 +#define REG_CAM0PLL_CSR 0x814 +#define REG_CAM1PLL_CSR 0x818 +#define REG_PLL_G2_SSC_SYN_CTRL 0x840 +#define REG_A0PLL_SSC_SYN_CTRL 0x850 +#define REG_A0PLL_SSC_SYN_SET 0x854 +#define REG_A0PLL_SSC_SYN_SPAN 0x858 +#define REG_A0PLL_SSC_SYN_STEP 0x85C +#define REG_DISPPLL_SSC_SYN_CTRL 0x860 +#define REG_DISPPLL_SSC_SYN_SET 0x864 +#define REG_DISPPLL_SSC_SYN_SPAN 0x868 +#define REG_DISPPLL_SSC_SYN_STEP 0x86C +#define REG_CAM0PLL_SSC_SYN_CTRL 0x870 +#define REG_CAM0PLL_SSC_SYN_SET 0x874 +#define REG_CAM0PLL_SSC_SYN_SPAN 0x878 +#define REG_CAM0PLL_SSC_SYN_STEP 0x87C +#define REG_CAM1PLL_SSC_SYN_CTRL 0x880 +#define REG_CAM1PLL_SSC_SYN_SET 0x884 +#define REG_CAM1PLL_SSC_SYN_SPAN 0x888 +#define REG_CAM1PLL_SSC_SYN_STEP 0x88C +#define REG_APLL_FRAC_DIV_CTRL 0x890 +#define REG_APLL_FRAC_DIV_M 0x894 +#define REG_APLL_FRAC_DIV_N 0x898 +#define REG_MIPIMPLL_CLK_CSR 0x8A0 +#define REG_A0PLL_CLK_CSR 0x8A4 +#define REG_DISPPLL_CLK_CSR 0x8A8 +#define REG_CAM0PLL_CLK_CSR 0x8AC +#define REG_CAM1PLL_CLK_CSR 0x8B0 +#define REG_CLK_CAM0_SRC_DIV 0x8C0 +#define REG_CLK_CAM1_SRC_DIV 0x8C4 + +/* top_pll_g6 */ +#define REG_PLL_G6_CTRL 0x900 +#define REG_PLL_G6_STATUS 0x904 +#define REG_MPLL_CSR 0x908 +#define REG_TPLL_CSR 0x90C +#define REG_FPLL_CSR 0x910 +#define REG_PLL_G6_SSC_SYN_CTRL 0x940 +#define REG_DPLL_SSC_SYN_CTRL 0x950 +#define REG_DPLL_SSC_SYN_SET 0x954 +#define REG_DPLL_SSC_SYN_SPAN 0x958 +#define REG_DPLL_SSC_SYN_STEP 0x95C +#define REG_MPLL_SSC_SYN_CTRL 0x960 +#define REG_MPLL_SSC_SYN_SET 0x964 +#define REG_MPLL_SSC_SYN_SPAN 0x968 +#define REG_MPLL_SSC_SYN_STEP 0x96C +#define REG_TPLL_SSC_SYN_CTRL 0x970 +#define REG_TPLL_SSC_SYN_SET 0x974 +#define REG_TPLL_SSC_SYN_SPAN 0x978 +#define REG_TPLL_SSC_SYN_STEP 0x97C + +/* clkgen */ +#define REG_CLK_EN_0 0x000 +#define REG_CLK_EN_1 0x004 +#define REG_CLK_EN_2 0x008 +#define REG_CLK_EN_3 0x00C +#define REG_CLK_EN_4 0x010 +#define REG_CLK_SEL_0 0x020 +#define REG_CLK_BYP_0 0x030 +#define REG_CLK_BYP_1 0x034 + +#define REG_DIV_CLK_A53_0 0x040 +#define REG_DIV_CLK_A53_1 0x044 +#define REG_DIV_CLK_CPU_AXI0 0x048 +#define REG_DIV_CLK_CPU_GIC 0x050 +#define REG_DIV_CLK_TPU 0x054 +#define REG_DIV_CLK_EMMC 0x064 +#define REG_DIV_CLK_EMMC_100K 0x06C +#define REG_DIV_CLK_SD0 0x070 +#define REG_DIV_CLK_SD0_100K 0x078 +#define REG_DIV_CLK_SD1 0x07C +#define REG_DIV_CLK_SD1_100K 0x084 +#define REG_DIV_CLK_SPI_NAND 0x088 +#define REG_DIV_CLK_ETH0_500M 0x08C +#define REG_DIV_CLK_ETH1_500M 0x090 +#define REG_DIV_CLK_GPIO_DB 0x094 +#define REG_DIV_CLK_SDMA_AUD0 0x098 +#define REG_DIV_CLK_SDMA_AUD1 0x09C +#define REG_DIV_CLK_SDMA_AUD2 0x0A0 +#define REG_DIV_CLK_SDMA_AUD3 0x0A4 +#define REG_DIV_CLK_CAM0_200 0x0A8 +#define REG_DIV_CLK_AXI4 0x0B8 +#define REG_DIV_CLK_AXI6 0x0BC +#define REG_DIV_CLK_DSI_ESC 0x0C4 +#define REG_DIV_CLK_AXI_VIP 0x0C8 +#define REG_DIV_CLK_SRC_VIP_SYS_0 0x0D0 +#define REG_DIV_CLK_SRC_VIP_SYS_1 0x0D8 +#define REG_DIV_CLK_DISP_SRC_VIP 0x0E0 +#define REG_DIV_CLK_AXI_VIDEO_CODEC 0x0E4 +#define REG_DIV_CLK_VC_SRC0 0x0EC +#define REG_DIV_CLK_1M 0x0FC +#define REG_DIV_CLK_SPI 0x100 +#define REG_DIV_CLK_I2C 0x104 +#define REG_DIV_CLK_SRC_VIP_SYS_2 0x110 +#define REG_DIV_CLK_AUDSRC 0x118 +#define REG_DIV_CLK_PWM_SRC_0 0x120 +#define REG_DIV_CLK_AP_DEBUG 0x128 +#define REG_DIV_CLK_RTCSYS_SRC_0 0x12C +#define REG_DIV_CLK_C906_0_0 0x130 +#define REG_DIV_CLK_C906_0_1 0x134 +#define REG_DIV_CLK_C906_1_0 0x138 +#define REG_DIV_CLK_C906_1_1 0x13C +#define REG_DIV_CLK_SRC_VIP_SYS_3 0x140 +#define REG_DIV_CLK_SRC_VIP_SYS_4 0x144 + +#endif /* _CLK_SOPHGO_CV1800_H_ */ diff --git a/drivers/clk/sophgo/clk-cv18xx-common.c b/drivers/clk/sophgo/clk-cv18xx-common.c new file mode 100644 index 000000000000..cbcdd88f0e23 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-common.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Inochi Amaoto <inochiama@outlook.com> + */ + +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/spinlock.h> +#include <linux/bug.h> + +#include "clk-cv18xx-common.h" + +int cv1800_clk_setbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field) +{ + u32 mask = BIT(field->shift); + u32 value; + unsigned long flags; + + spin_lock_irqsave(common->lock, flags); + + value = readl(common->base + field->reg); + writel(value | mask, common->base + field->reg); + + spin_unlock_irqrestore(common->lock, flags); + + return 0; +} + +int cv1800_clk_clearbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field) +{ + u32 mask = BIT(field->shift); + u32 value; + unsigned long flags; + + spin_lock_irqsave(common->lock, flags); + + value = readl(common->base + field->reg); + writel(value & ~mask, common->base + field->reg); + + spin_unlock_irqrestore(common->lock, flags); + + return 0; +} + +int cv1800_clk_checkbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field) +{ + return readl(common->base + field->reg) & BIT(field->shift); +} + +#define PLL_LOCK_TIMEOUT_US (200 * 1000) + +void cv1800_clk_wait_for_lock(struct cv1800_clk_common *common, + u32 reg, u32 lock) +{ + void __iomem *addr = common->base + reg; + u32 regval; + + if (!lock) + return; + + WARN_ON(readl_relaxed_poll_timeout(addr, regval, regval & lock, + 100, PLL_LOCK_TIMEOUT_US)); +} diff --git a/drivers/clk/sophgo/clk-cv18xx-common.h b/drivers/clk/sophgo/clk-cv18xx-common.h new file mode 100644 index 000000000000..2bfda02b2064 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-common.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023 Inochi Amaoto <inochiama@outlook.com> + */ + +#ifndef _CLK_SOPHGO_CV18XX_IP_H_ +#define _CLK_SOPHGO_CV18XX_IP_H_ + +#include <linux/compiler.h> +#include <linux/clk-provider.h> +#include <linux/bitfield.h> + +struct cv1800_clk_common { + void __iomem *base; + spinlock_t *lock; + struct clk_hw hw; + unsigned long features; +}; + +#define CV1800_CLK_COMMON(_name, _parents, _op, _flags) \ + { \ + .hw.init = CLK_HW_INIT_PARENTS_DATA(_name, _parents, \ + _op, _flags), \ + } + +static inline struct cv1800_clk_common * +hw_to_cv1800_clk_common(struct clk_hw *hw) +{ + return container_of(hw, struct cv1800_clk_common, hw); +} + +struct cv1800_clk_regbit { + u16 reg; + s8 shift; +}; + +struct cv1800_clk_regfield { + u16 reg; + u8 shift; + u8 width; + s16 initval; + unsigned long flags; +}; + +#define CV1800_CLK_BIT(_reg, _shift) \ + { \ + .reg = _reg, \ + .shift = _shift, \ + } + +#define CV1800_CLK_REG(_reg, _shift, _width, _initval, _flags) \ + { \ + .reg = _reg, \ + .shift = _shift, \ + .width = _width, \ + .initval = _initval, \ + .flags = _flags, \ + } + +#define cv1800_clk_regfield_genmask(_reg) \ + GENMASK((_reg)->shift + (_reg)->width - 1, (_reg)->shift) +#define cv1800_clk_regfield_get(_val, _reg) \ + (((_val) >> (_reg)->shift) & GENMASK((_reg)->width - 1, 0)) +#define cv1800_clk_regfield_set(_val, _new, _reg) \ + (((_val) & ~cv1800_clk_regfield_genmask((_reg))) | \ + (((_new) & GENMASK((_reg)->width - 1, 0)) << (_reg)->shift)) + +#define _CV1800_SET_FIELD(_reg, _val, _field) \ + (((_reg) & ~(_field)) | FIELD_PREP((_field), (_val))) + +int cv1800_clk_setbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field); +int cv1800_clk_clearbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field); +int cv1800_clk_checkbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field); + +void cv1800_clk_wait_for_lock(struct cv1800_clk_common *common, + u32 reg, u32 lock); + +#endif // _CLK_SOPHGO_CV18XX_IP_H_ diff --git a/drivers/clk/sophgo/clk-cv18xx-ip.c b/drivers/clk/sophgo/clk-cv18xx-ip.c new file mode 100644 index 000000000000..805f561725ae --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-ip.c @@ -0,0 +1,887 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Inochi Amaoto <inochiama@outlook.com> + */ + +#include <linux/clk-provider.h> +#include <linux/io.h> +#include <linux/gcd.h> +#include <linux/spinlock.h> + +#include "clk-cv18xx-ip.h" + +/* GATE */ +static inline struct cv1800_clk_gate *hw_to_cv1800_clk_gate(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_gate, common); +} + +static int gate_enable(struct clk_hw *hw) +{ + struct cv1800_clk_gate *gate = hw_to_cv1800_clk_gate(hw); + + return cv1800_clk_setbit(&gate->common, &gate->gate); +} + +static void gate_disable(struct clk_hw *hw) +{ + struct cv1800_clk_gate *gate = hw_to_cv1800_clk_gate(hw); + + cv1800_clk_clearbit(&gate->common, &gate->gate); +} + +static int gate_is_enabled(struct clk_hw *hw) +{ + struct cv1800_clk_gate *gate = hw_to_cv1800_clk_gate(hw); + + return cv1800_clk_checkbit(&gate->common, &gate->gate); +} + +static unsigned long gate_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + return parent_rate; +} + +static long gate_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + return *parent_rate; +} + +static int gate_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + return 0; +} + +const struct clk_ops cv1800_clk_gate_ops = { + .disable = gate_disable, + .enable = gate_enable, + .is_enabled = gate_is_enabled, + + .recalc_rate = gate_recalc_rate, + .round_rate = gate_round_rate, + .set_rate = gate_set_rate, +}; + +/* DIV */ +#define _DIV_EN_CLK_DIV_FACTOR_FIELD BIT(3) + +#define DIV_GET_EN_CLK_DIV_FACTOR(_reg) \ + FIELD_GET(_DIV_EN_CLK_DIV_FACTOR_FIELD, _reg) + +#define DIV_SET_EN_DIV_FACTOR(_reg) \ + _CV1800_SET_FIELD(_reg, 1, _DIV_EN_CLK_DIV_FACTOR_FIELD) + +static inline struct cv1800_clk_div *hw_to_cv1800_clk_div(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_div, common); +} + +static int div_enable(struct clk_hw *hw) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + + return cv1800_clk_setbit(&div->common, &div->gate); +} + +static void div_disable(struct clk_hw *hw) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + + cv1800_clk_clearbit(&div->common, &div->gate); +} + +static int div_is_enabled(struct clk_hw *hw) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + + return cv1800_clk_checkbit(&div->common, &div->gate); +} + +static int div_helper_set_rate(struct cv1800_clk_common *common, + struct cv1800_clk_regfield *div, + unsigned long val) +{ + unsigned long flags; + u32 reg; + + if (div->width == 0) + return 0; + + spin_lock_irqsave(common->lock, flags); + + reg = readl(common->base + div->reg); + reg = cv1800_clk_regfield_set(reg, val, div); + if (div->initval > 0) + reg = DIV_SET_EN_DIV_FACTOR(reg); + + writel(reg, common->base + div->reg); + + spin_unlock_irqrestore(common->lock, flags); + + return 0; +} + +static u32 div_helper_get_clockdiv(struct cv1800_clk_common *common, + struct cv1800_clk_regfield *div) +{ + u32 clockdiv = 1; + u32 reg; + + if (!div || div->initval < 0 || (div->width == 0 && div->initval <= 0)) + return 1; + + if (div->width == 0 && div->initval > 0) + return div->initval; + + reg = readl(common->base + div->reg); + + if (div->initval == 0 || DIV_GET_EN_CLK_DIV_FACTOR(reg)) + clockdiv = cv1800_clk_regfield_get(reg, div); + else if (div->initval > 0) + clockdiv = div->initval; + + return clockdiv; +} + +static u32 div_helper_round_rate(struct cv1800_clk_regfield *div, + struct clk_hw *hw, struct clk_hw *parent, + unsigned long rate, unsigned long *prate) +{ + if (div->width == 0) { + if (div->initval <= 0) + return DIV_ROUND_UP_ULL(*prate, 1); + else + return DIV_ROUND_UP_ULL(*prate, div->initval); + } + + return divider_round_rate_parent(hw, parent, rate, prate, NULL, + div->width, div->flags); +} + +static long div_round_rate(struct clk_hw *parent, unsigned long *parent_rate, + unsigned long rate, int id, void *data) +{ + struct cv1800_clk_div *div = data; + + return div_helper_round_rate(&div->div, &div->common.hw, parent, + rate, parent_rate); +} + +static bool div_is_better_rate(struct cv1800_clk_common *common, + unsigned long target, unsigned long now, + unsigned long best) +{ + if (common->features & CLK_DIVIDER_ROUND_CLOSEST) + return abs_diff(target, now) < abs_diff(target, best); + + return now <= target && now > best; +} + +static int mux_helper_determine_rate(struct cv1800_clk_common *common, + struct clk_rate_request *req, + long (*round)(struct clk_hw *, + unsigned long *, + unsigned long, + int, + void *), + void *data) +{ + unsigned long best_parent_rate = 0, best_rate = 0; + struct clk_hw *best_parent, *hw = &common->hw; + unsigned int i; + + if (clk_hw_get_flags(hw) & CLK_SET_RATE_NO_REPARENT) { + unsigned long adj_parent_rate; + + best_parent = clk_hw_get_parent(hw); + best_parent_rate = clk_hw_get_rate(best_parent); + + best_rate = round(best_parent, &adj_parent_rate, + req->rate, -1, data); + + goto find; + } + + for (i = 0; i < clk_hw_get_num_parents(hw); i++) { + unsigned long tmp_rate, parent_rate; + struct clk_hw *parent; + + parent = clk_hw_get_parent_by_index(hw, i); + if (!parent) + continue; + + parent_rate = clk_hw_get_rate(parent); + + tmp_rate = round(parent, &parent_rate, req->rate, i, data); + + if (tmp_rate == req->rate) { + best_parent = parent; + best_parent_rate = parent_rate; + best_rate = tmp_rate; + goto find; + } + + if (div_is_better_rate(common, req->rate, + tmp_rate, best_rate)) { + best_parent = parent; + best_parent_rate = parent_rate; + best_rate = tmp_rate; + } + } + + if (best_rate == 0) + return -EINVAL; + +find: + req->best_parent_hw = best_parent; + req->best_parent_rate = best_parent_rate; + req->rate = best_rate; + return 0; +} + +static int div_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + + return mux_helper_determine_rate(&div->common, req, + div_round_rate, div); +} + +static unsigned long div_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + unsigned long val; + + val = div_helper_get_clockdiv(&div->common, &div->div); + if (val == 0) + return 0; + + return divider_recalc_rate(hw, parent_rate, val, NULL, + div->div.flags, div->div.width); +} + +static int div_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + unsigned long val; + + val = divider_get_val(rate, parent_rate, NULL, + div->div.width, div->div.flags); + + return div_helper_set_rate(&div->common, &div->div, val); +} + +const struct clk_ops cv1800_clk_div_ops = { + .disable = div_disable, + .enable = div_enable, + .is_enabled = div_is_enabled, + + .determine_rate = div_determine_rate, + .recalc_rate = div_recalc_rate, + .set_rate = div_set_rate, +}; + +static inline struct cv1800_clk_bypass_div * +hw_to_cv1800_clk_bypass_div(struct clk_hw *hw) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + + return container_of(div, struct cv1800_clk_bypass_div, div); +} + +static long bypass_div_round_rate(struct clk_hw *parent, + unsigned long *parent_rate, + unsigned long rate, int id, void *data) +{ + struct cv1800_clk_bypass_div *div = data; + + if (id == -1) { + if (cv1800_clk_checkbit(&div->div.common, &div->bypass)) + return *parent_rate; + else + return div_round_rate(parent, parent_rate, rate, + -1, &div->div); + } + + if (id == 0) + return *parent_rate; + + return div_round_rate(parent, parent_rate, rate, id - 1, &div->div); +} + +static int bypass_div_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_bypass_div *div = hw_to_cv1800_clk_bypass_div(hw); + + return mux_helper_determine_rate(&div->div.common, req, + bypass_div_round_rate, div); +} + +static unsigned long bypass_div_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_bypass_div *div = hw_to_cv1800_clk_bypass_div(hw); + + if (cv1800_clk_checkbit(&div->div.common, &div->bypass)) + return parent_rate; + + return div_recalc_rate(hw, parent_rate); +} + +static int bypass_div_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_bypass_div *div = hw_to_cv1800_clk_bypass_div(hw); + + if (cv1800_clk_checkbit(&div->div.common, &div->bypass)) + return 0; + + return div_set_rate(hw, rate, parent_rate); +} + +static u8 bypass_div_get_parent(struct clk_hw *hw) +{ + struct cv1800_clk_bypass_div *div = hw_to_cv1800_clk_bypass_div(hw); + + if (cv1800_clk_checkbit(&div->div.common, &div->bypass)) + return 0; + + return 1; +} + +static int bypass_div_set_parent(struct clk_hw *hw, u8 index) +{ + struct cv1800_clk_bypass_div *div = hw_to_cv1800_clk_bypass_div(hw); + + if (index) + return cv1800_clk_clearbit(&div->div.common, &div->bypass); + + return cv1800_clk_setbit(&div->div.common, &div->bypass); +} + +const struct clk_ops cv1800_clk_bypass_div_ops = { + .disable = div_disable, + .enable = div_enable, + .is_enabled = div_is_enabled, + + .determine_rate = bypass_div_determine_rate, + .recalc_rate = bypass_div_recalc_rate, + .set_rate = bypass_div_set_rate, + + .set_parent = bypass_div_set_parent, + .get_parent = bypass_div_get_parent, +}; + +/* MUX */ +static inline struct cv1800_clk_mux *hw_to_cv1800_clk_mux(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_mux, common); +} + +static int mux_enable(struct clk_hw *hw) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + + return cv1800_clk_setbit(&mux->common, &mux->gate); +} + +static void mux_disable(struct clk_hw *hw) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + + cv1800_clk_clearbit(&mux->common, &mux->gate); +} + +static int mux_is_enabled(struct clk_hw *hw) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + + return cv1800_clk_checkbit(&mux->common, &mux->gate); +} + +static long mux_round_rate(struct clk_hw *parent, unsigned long *parent_rate, + unsigned long rate, int id, void *data) +{ + struct cv1800_clk_mux *mux = data; + + return div_helper_round_rate(&mux->div, &mux->common.hw, parent, + rate, parent_rate); +} + +static int mux_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + + return mux_helper_determine_rate(&mux->common, req, + mux_round_rate, mux); +} + +static unsigned long mux_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + unsigned long val; + + val = div_helper_get_clockdiv(&mux->common, &mux->div); + if (val == 0) + return 0; + + return divider_recalc_rate(hw, parent_rate, val, NULL, + mux->div.flags, mux->div.width); +} + +static int mux_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + unsigned long val; + + val = divider_get_val(rate, parent_rate, NULL, + mux->div.width, mux->div.flags); + + return div_helper_set_rate(&mux->common, &mux->div, val); +} + +static u8 mux_get_parent(struct clk_hw *hw) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + u32 reg = readl(mux->common.base + mux->mux.reg); + + return cv1800_clk_regfield_get(reg, &mux->mux); +} + +static int _mux_set_parent(struct cv1800_clk_mux *mux, u8 index) +{ + u32 reg; + + reg = readl(mux->common.base + mux->mux.reg); + reg = cv1800_clk_regfield_set(reg, index, &mux->mux); + writel(reg, mux->common.base + mux->mux.reg); + + return 0; +} + +static int mux_set_parent(struct clk_hw *hw, u8 index) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + unsigned long flags; + + spin_lock_irqsave(mux->common.lock, flags); + + _mux_set_parent(mux, index); + + spin_unlock_irqrestore(mux->common.lock, flags); + + return 0; +} + +const struct clk_ops cv1800_clk_mux_ops = { + .disable = mux_disable, + .enable = mux_enable, + .is_enabled = mux_is_enabled, + + .determine_rate = mux_determine_rate, + .recalc_rate = mux_recalc_rate, + .set_rate = mux_set_rate, + + .set_parent = mux_set_parent, + .get_parent = mux_get_parent, +}; + +static inline struct cv1800_clk_bypass_mux * +hw_to_cv1800_clk_bypass_mux(struct clk_hw *hw) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + + return container_of(mux, struct cv1800_clk_bypass_mux, mux); +} + +static long bypass_mux_round_rate(struct clk_hw *parent, + unsigned long *parent_rate, + unsigned long rate, int id, void *data) +{ + struct cv1800_clk_bypass_mux *mux = data; + + if (id == -1) { + if (cv1800_clk_checkbit(&mux->mux.common, &mux->bypass)) + return *parent_rate; + else + return mux_round_rate(parent, parent_rate, rate, + -1, &mux->mux); + } + + if (id == 0) + return *parent_rate; + + return mux_round_rate(parent, parent_rate, rate, id - 1, &mux->mux); +} + +static int bypass_mux_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_bypass_mux *mux = hw_to_cv1800_clk_bypass_mux(hw); + + return mux_helper_determine_rate(&mux->mux.common, req, + bypass_mux_round_rate, mux); +} + +static unsigned long bypass_mux_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_bypass_mux *mux = hw_to_cv1800_clk_bypass_mux(hw); + + if (cv1800_clk_checkbit(&mux->mux.common, &mux->bypass)) + return parent_rate; + + return mux_recalc_rate(hw, parent_rate); +} + +static int bypass_mux_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_bypass_mux *mux = hw_to_cv1800_clk_bypass_mux(hw); + + if (cv1800_clk_checkbit(&mux->mux.common, &mux->bypass)) + return 0; + + return mux_set_rate(hw, rate, parent_rate); +} + +static u8 bypass_mux_get_parent(struct clk_hw *hw) +{ + struct cv1800_clk_bypass_mux *mux = hw_to_cv1800_clk_bypass_mux(hw); + + if (cv1800_clk_checkbit(&mux->mux.common, &mux->bypass)) + return 0; + + return mux_get_parent(hw) + 1; +} + +static int bypass_mux_set_parent(struct clk_hw *hw, u8 index) +{ + struct cv1800_clk_bypass_mux *mux = hw_to_cv1800_clk_bypass_mux(hw); + + if (index == 0) + return cv1800_clk_setbit(&mux->mux.common, &mux->bypass); + + return cv1800_clk_clearbit(&mux->mux.common, &mux->bypass); +} + +const struct clk_ops cv1800_clk_bypass_mux_ops = { + .disable = mux_disable, + .enable = mux_enable, + .is_enabled = mux_is_enabled, + + .determine_rate = bypass_mux_determine_rate, + .recalc_rate = bypass_mux_recalc_rate, + .set_rate = bypass_mux_set_rate, + + .set_parent = bypass_mux_set_parent, + .get_parent = bypass_mux_get_parent, +}; + +/* MMUX */ +static inline struct cv1800_clk_mmux *hw_to_cv1800_clk_mmux(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_mmux, common); +} + +static u8 mmux_get_parent_id(struct cv1800_clk_mmux *mmux) +{ + struct clk_hw *hw = &mmux->common.hw; + struct clk_hw *parent = clk_hw_get_parent(hw); + unsigned int i; + + for (i = 0; i < clk_hw_get_num_parents(hw); i++) { + if (parent == clk_hw_get_parent_by_index(hw, i)) + return i; + } + + unreachable(); +} + +static int mmux_enable(struct clk_hw *hw) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + + return cv1800_clk_setbit(&mmux->common, &mmux->gate); +} + +static void mmux_disable(struct clk_hw *hw) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + + cv1800_clk_clearbit(&mmux->common, &mmux->gate); +} + +static int mmux_is_enabled(struct clk_hw *hw) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + + return cv1800_clk_checkbit(&mmux->common, &mmux->gate); +} + +static long mmux_round_rate(struct clk_hw *parent, unsigned long *parent_rate, + unsigned long rate, int id, void *data) +{ + struct cv1800_clk_mmux *mmux = data; + s8 div_id; + + if (id == -1) { + if (cv1800_clk_checkbit(&mmux->common, &mmux->bypass)) + return *parent_rate; + + id = mmux_get_parent_id(mmux); + } + + div_id = mmux->parent2sel[id]; + + if (div_id < 0) + return *parent_rate; + + return div_helper_round_rate(&mmux->div[div_id], + &mmux->common.hw, parent, + rate, parent_rate); +} + +static int mmux_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + + return mux_helper_determine_rate(&mmux->common, req, + mmux_round_rate, mmux); +} + +static unsigned long mmux_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + unsigned long val; + struct cv1800_clk_regfield *div; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->bypass)) + return parent_rate; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->clk_sel)) + div = &mmux->div[0]; + else + div = &mmux->div[1]; + + val = div_helper_get_clockdiv(&mmux->common, div); + if (val == 0) + return 0; + + return divider_recalc_rate(hw, parent_rate, val, NULL, + div->flags, div->width); +} + +static int mmux_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + struct cv1800_clk_regfield *div; + unsigned long val; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->bypass)) + return parent_rate; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->clk_sel)) + div = &mmux->div[0]; + else + div = &mmux->div[1]; + + val = divider_get_val(rate, parent_rate, NULL, + div->width, div->flags); + + return div_helper_set_rate(&mmux->common, div, val); +} + +static u8 mmux_get_parent(struct clk_hw *hw) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + struct cv1800_clk_regfield *mux; + u32 reg; + s8 clk_sel; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->bypass)) + return 0; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->clk_sel)) + clk_sel = 0; + else + clk_sel = 1; + mux = &mmux->mux[clk_sel]; + + reg = readl(mmux->common.base + mux->reg); + + return mmux->sel2parent[clk_sel][cv1800_clk_regfield_get(reg, mux)]; +} + +static int mmux_set_parent(struct clk_hw *hw, u8 index) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + struct cv1800_clk_regfield *mux; + unsigned long flags; + u32 reg; + s8 clk_sel = mmux->parent2sel[index]; + + if (index == 0 || clk_sel == -1) { + cv1800_clk_setbit(&mmux->common, &mmux->bypass); + goto release; + } + + cv1800_clk_clearbit(&mmux->common, &mmux->bypass); + + if (clk_sel) + cv1800_clk_clearbit(&mmux->common, &mmux->clk_sel); + else + cv1800_clk_setbit(&mmux->common, &mmux->clk_sel); + + spin_lock_irqsave(mmux->common.lock, flags); + + mux = &mmux->mux[clk_sel]; + reg = readl(mmux->common.base + mux->reg); + reg = cv1800_clk_regfield_set(reg, index, mux); + + writel(reg, mmux->common.base + mux->reg); + + spin_unlock_irqrestore(mmux->common.lock, flags); + +release: + return 0; +} + +const struct clk_ops cv1800_clk_mmux_ops = { + .disable = mmux_disable, + .enable = mmux_enable, + .is_enabled = mmux_is_enabled, + + .determine_rate = mmux_determine_rate, + .recalc_rate = mmux_recalc_rate, + .set_rate = mmux_set_rate, + + .set_parent = mmux_set_parent, + .get_parent = mmux_get_parent, +}; + +/* AUDIO CLK */ +static inline struct cv1800_clk_audio * +hw_to_cv1800_clk_audio(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_audio, common); +} + +static int aclk_enable(struct clk_hw *hw) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + + cv1800_clk_setbit(&aclk->common, &aclk->src_en); + return cv1800_clk_setbit(&aclk->common, &aclk->output_en); +} + +static void aclk_disable(struct clk_hw *hw) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + + cv1800_clk_clearbit(&aclk->common, &aclk->output_en); + cv1800_clk_clearbit(&aclk->common, &aclk->src_en); +} + +static int aclk_is_enabled(struct clk_hw *hw) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + + return cv1800_clk_checkbit(&aclk->common, &aclk->output_en); +} + +static int aclk_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + + req->rate = aclk->target_rate; + + return 0; +} + +static unsigned long aclk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + u64 rate = parent_rate; + u64 factor = 2; + u32 regval; + + if (!cv1800_clk_checkbit(&aclk->common, &aclk->div_en)) + return 0; + + regval = readl(aclk->common.base + aclk->m.reg); + factor *= cv1800_clk_regfield_get(regval, &aclk->m); + + regval = readl(aclk->common.base + aclk->n.reg); + rate *= cv1800_clk_regfield_get(regval, &aclk->n); + + return DIV64_U64_ROUND_UP(rate, factor); +} + +static void aclk_determine_mn(unsigned long parent_rate, unsigned long rate, + u32 *m, u32 *n) +{ + u32 tm = parent_rate / 2; + u32 tn = rate; + u32 tcommon = gcd(tm, tn); + *m = tm / tcommon; + *n = tn / tcommon; +} + +static int aclk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + unsigned long flags; + u32 m, n; + + aclk_determine_mn(parent_rate, rate, + &m, &n); + + spin_lock_irqsave(aclk->common.lock, flags); + + writel(m, aclk->common.base + aclk->m.reg); + writel(n, aclk->common.base + aclk->n.reg); + + cv1800_clk_setbit(&aclk->common, &aclk->div_en); + cv1800_clk_setbit(&aclk->common, &aclk->div_up); + + spin_unlock_irqrestore(aclk->common.lock, flags); + + return 0; +} + +const struct clk_ops cv1800_clk_audio_ops = { + .disable = aclk_disable, + .enable = aclk_enable, + .is_enabled = aclk_is_enabled, + + .determine_rate = aclk_determine_rate, + .recalc_rate = aclk_recalc_rate, + .set_rate = aclk_set_rate, +}; diff --git a/drivers/clk/sophgo/clk-cv18xx-ip.h b/drivers/clk/sophgo/clk-cv18xx-ip.h new file mode 100644 index 000000000000..b37ba42bfde3 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-ip.h @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023 Inochi Amaoto <inochiama@outlook.com> + */ + +#ifndef _CLK_SOPHGO_CV1800_IP_H_ +#define _CLK_SOPHGO_CV1800_IP_H_ + +#include "clk-cv18xx-common.h" + +struct cv1800_clk_gate { + struct cv1800_clk_common common; + struct cv1800_clk_regbit gate; +}; + +struct cv1800_clk_div_data { + u32 reg; + u32 mask; + u32 width; + u32 init; + u32 flags; +}; + +struct cv1800_clk_div { + struct cv1800_clk_common common; + struct cv1800_clk_regbit gate; + struct cv1800_clk_regfield div; +}; + +struct cv1800_clk_bypass_div { + struct cv1800_clk_div div; + struct cv1800_clk_regbit bypass; +}; + +struct cv1800_clk_mux { + struct cv1800_clk_common common; + struct cv1800_clk_regbit gate; + struct cv1800_clk_regfield div; + struct cv1800_clk_regfield mux; +}; + +struct cv1800_clk_bypass_mux { + struct cv1800_clk_mux mux; + struct cv1800_clk_regbit bypass; +}; + +struct cv1800_clk_mmux { + struct cv1800_clk_common common; + struct cv1800_clk_regbit gate; + struct cv1800_clk_regfield div[2]; + struct cv1800_clk_regfield mux[2]; + struct cv1800_clk_regbit bypass; + struct cv1800_clk_regbit clk_sel; + const s8 *parent2sel; + const u8 *sel2parent[2]; +}; + +struct cv1800_clk_audio { + struct cv1800_clk_common common; + struct cv1800_clk_regbit src_en; + struct cv1800_clk_regbit output_en; + struct cv1800_clk_regbit div_en; + struct cv1800_clk_regbit div_up; + struct cv1800_clk_regfield m; + struct cv1800_clk_regfield n; + u32 target_rate; +}; + +#define CV1800_GATE(_name, _parent, _gate_reg, _gate_shift, _flags) \ + struct cv1800_clk_gate _name = { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + &cv1800_clk_gate_ops, \ + _flags), \ + .gate = CV1800_CLK_BIT(_gate_reg, _gate_shift), \ + } + +#define _CV1800_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, _ops, _flags) \ + { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + _ops, _flags), \ + .gate = CV1800_CLK_BIT(_gate_reg, \ + _gate_shift), \ + .div = CV1800_CLK_REG(_div_reg, _div_shift, \ + _div_width, _div_init, \ + _div_flag), \ + } + +#define _CV1800_FIXED_DIV_FLAG \ + (CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ROUND_CLOSEST) + +#define _CV1800_FIXED_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _fix_div, _ops, _flags) \ + { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + _ops, _flags), \ + .gate = CV1800_CLK_BIT(_gate_reg, \ + _gate_shift), \ + .div = CV1800_CLK_REG(0, 0, 0, \ + _fix_div, \ + _CV1800_FIXED_DIV_FLAG),\ + } + +#define CV1800_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, _flags) \ + struct cv1800_clk_div _name = \ + _CV1800_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init,\ + _div_flag, &cv1800_clk_div_ops, _flags) + +#define CV1800_BYPASS_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, _bypass_reg, _bypass_shift, _flags)\ + struct cv1800_clk_bypass_div _name = { \ + .div = _CV1800_DIV(_name, _parent, \ + _gate_reg, _gate_shift, \ + _div_reg, _div_shift, \ + _div_width, _div_init, _div_flag, \ + &cv1800_clk_bypass_div_ops, \ + _flags), \ + .bypass = CV1800_CLK_BIT(_bypass_reg, _bypass_shift), \ + } + +#define CV1800_FIXED_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _fix_div, _flags) \ + struct cv1800_clk_div _name = \ + _CV1800_FIXED_DIV(_name, _parent, \ + _gate_reg, _gate_shift, \ + _fix_div, \ + &cv1800_clk_div_ops, _flags) \ + +#define CV1800_BYPASS_FIXED_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _fix_div, _bypass_reg, _bypass_shift, \ + _flags) \ + struct cv1800_clk_bypass_div _name = { \ + .div = _CV1800_FIXED_DIV(_name, _parent, \ + _gate_reg, _gate_shift, \ + _fix_div, \ + &cv1800_clk_bypass_div_ops, \ + _flags), \ + .bypass = CV1800_CLK_BIT(_bypass_reg, _bypass_shift), \ + } + +#define _CV1800_MUX(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, \ + _mux_reg, _mux_shift, _mux_width, \ + _ops, _flags) \ + { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + _ops, _flags), \ + .gate = CV1800_CLK_BIT(_gate_reg, \ + _gate_shift), \ + .div = CV1800_CLK_REG(_div_reg, _div_shift, \ + _div_width, _div_init, \ + _div_flag), \ + .mux = CV1800_CLK_REG(_mux_reg, _mux_shift, \ + _mux_width, 0, 0), \ + } + +#define CV1800_MUX(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, \ + _mux_reg, _mux_shift, _mux_width, _flags) \ + struct cv1800_clk_mux _name = \ + _CV1800_MUX(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init,\ + _div_flag, _mux_reg, _mux_shift, _mux_width,\ + &cv1800_clk_mux_ops, _flags) + +#define CV1800_BYPASS_MUX(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, \ + _mux_reg, _mux_shift, _mux_width, \ + _bypass_reg, _bypass_shift, _flags) \ + struct cv1800_clk_bypass_mux _name = { \ + .mux = _CV1800_MUX(_name, _parent, \ + _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, \ + _div_init, _div_flag, \ + _mux_reg, _mux_shift, _mux_width, \ + &cv1800_clk_bypass_mux_ops, \ + _flags), \ + .bypass = CV1800_CLK_BIT(_bypass_reg, _bypass_shift), \ + } + +#define CV1800_MMUX(_name, _parent, _gate_reg, _gate_shift, \ + _div0_reg, _div0_shift, _div0_width, _div0_init, \ + _div0_flag, \ + _div1_reg, _div1_shift, _div1_width, _div1_init, \ + _div1_flag, \ + _mux0_reg, _mux0_shift, _mux0_width, \ + _mux1_reg, _mux1_shift, _mux1_width, \ + _bypass_reg, _bypass_shift, \ + _clk_sel_reg, _clk_sel_shift, \ + _parent2sel, _sel2parent0, _sel2parent1, _flags) \ + struct cv1800_clk_mmux _name = { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + &cv1800_clk_mmux_ops,\ + _flags), \ + .gate = CV1800_CLK_BIT(_gate_reg, _gate_shift),\ + .div = { \ + CV1800_CLK_REG(_div0_reg, _div0_shift, \ + _div0_width, _div0_init, \ + _div0_flag), \ + CV1800_CLK_REG(_div1_reg, _div1_shift, \ + _div1_width, _div1_init, \ + _div1_flag), \ + }, \ + .mux = { \ + CV1800_CLK_REG(_mux0_reg, _mux0_shift, \ + _mux0_width, 0, 0), \ + CV1800_CLK_REG(_mux1_reg, _mux1_shift, \ + _mux1_width, 0, 0), \ + }, \ + .bypass = CV1800_CLK_BIT(_bypass_reg, \ + _bypass_shift), \ + .clk_sel = CV1800_CLK_BIT(_clk_sel_reg, \ + _clk_sel_shift), \ + .parent2sel = _parent2sel, \ + .sel2parent = { _sel2parent0, _sel2parent1 }, \ + } + +#define CV1800_ACLK(_name, _parent, \ + _src_en_reg, _src_en_reg_shift, \ + _output_en_reg, _output_en_shift, \ + _div_en_reg, _div_en_reg_shift, \ + _div_up_reg, _div_up_reg_shift, \ + _m_reg, _m_shift, _m_width, _m_flag, \ + _n_reg, _n_shift, _n_width, _n_flag, \ + _target_rate, _flags) \ + struct cv1800_clk_audio _name = { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + &cv1800_clk_audio_ops,\ + _flags), \ + .src_en = CV1800_CLK_BIT(_src_en_reg, \ + _src_en_reg_shift), \ + .output_en = CV1800_CLK_BIT(_output_en_reg, \ + _output_en_shift), \ + .div_en = CV1800_CLK_BIT(_div_en_reg, \ + _div_en_reg_shift), \ + .div_up = CV1800_CLK_BIT(_div_up_reg, \ + _div_up_reg_shift), \ + .m = CV1800_CLK_REG(_m_reg, _m_shift, \ + _m_width, 0, _m_flag), \ + .n = CV1800_CLK_REG(_n_reg, _n_shift, \ + _n_width, 0, _n_flag), \ + .target_rate = _target_rate, \ + } + +extern const struct clk_ops cv1800_clk_gate_ops; +extern const struct clk_ops cv1800_clk_div_ops; +extern const struct clk_ops cv1800_clk_bypass_div_ops; +extern const struct clk_ops cv1800_clk_mux_ops; +extern const struct clk_ops cv1800_clk_bypass_mux_ops; +extern const struct clk_ops cv1800_clk_mmux_ops; +extern const struct clk_ops cv1800_clk_audio_ops; + +#endif // _CLK_SOPHGO_CV1800_IP_H_ diff --git a/drivers/clk/sophgo/clk-cv18xx-pll.c b/drivers/clk/sophgo/clk-cv18xx-pll.c new file mode 100644 index 000000000000..29e24098bf5f --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-pll.c @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Inochi Amaoto <inochiama@outlook.com> + */ + +#include <linux/clk-provider.h> +#include <linux/io.h> +#include <linux/limits.h> +#include <linux/spinlock.h> + +#include "clk-cv18xx-pll.h" + +static inline struct cv1800_clk_pll *hw_to_cv1800_clk_pll(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_pll, common); +} + +static unsigned long ipll_calc_rate(unsigned long parent_rate, + unsigned long pre_div_sel, + unsigned long div_sel, + unsigned long post_div_sel) +{ + uint64_t rate = parent_rate; + + rate *= div_sel; + do_div(rate, pre_div_sel * post_div_sel); + + return rate; +} + +static unsigned long ipll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + u32 value; + + value = readl(pll->common.base + pll->pll_reg); + + return ipll_calc_rate(parent_rate, + PLL_GET_PRE_DIV_SEL(value), + PLL_GET_DIV_SEL(value), + PLL_GET_POST_DIV_SEL(value)); +} + +static int ipll_find_rate(const struct cv1800_clk_pll_limit *limit, + unsigned long prate, unsigned long *rate, + u32 *value) +{ + unsigned long best_rate = 0; + unsigned long trate = *rate; + unsigned long pre_div_sel = 0, div_sel = 0, post_div_sel = 0; + unsigned long pre, div, post; + u32 detected = *value; + unsigned long tmp; + + for_each_pll_limit_range(pre, &limit->pre_div) { + for_each_pll_limit_range(div, &limit->div) { + for_each_pll_limit_range(post, &limit->post_div) { + tmp = ipll_calc_rate(prate, pre, div, post); + + if (tmp > trate) + continue; + + if ((trate - tmp) < (trate - best_rate)) { + best_rate = tmp; + pre_div_sel = pre; + div_sel = div; + post_div_sel = post; + } + } + } + } + + if (best_rate) { + detected = PLL_SET_PRE_DIV_SEL(detected, pre_div_sel); + detected = PLL_SET_POST_DIV_SEL(detected, post_div_sel); + detected = PLL_SET_DIV_SEL(detected, div_sel); + *value = detected; + *rate = best_rate; + return 0; + } + + return -EINVAL; +} + +static int ipll_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) +{ + u32 val; + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + return ipll_find_rate(pll->pll_limit, req->best_parent_rate, + &req->rate, &val); +} + +static void pll_get_mode_ctrl(unsigned long div_sel, + bool (*mode_ctrl_check)(unsigned long, + unsigned long, + unsigned long), + const struct cv1800_clk_pll_limit *limit, + u32 *value) +{ + unsigned long ictrl = 0, mode = 0; + u32 detected = *value; + + for_each_pll_limit_range(mode, &limit->mode) { + for_each_pll_limit_range(ictrl, &limit->ictrl) { + if (mode_ctrl_check(div_sel, ictrl, mode)) { + detected = PLL_SET_SEL_MODE(detected, mode); + detected = PLL_SET_ICTRL(detected, ictrl); + *value = detected; + return; + } + } + } +} + +static bool ipll_check_mode_ctrl_restrict(unsigned long div_sel, + unsigned long ictrl, + unsigned long mode) +{ + unsigned long left_rest = 20 * div_sel; + unsigned long right_rest = 35 * div_sel; + unsigned long test = 184 * (1 + mode) * (1 + ictrl) / 2; + + return test > left_rest && test <= right_rest; +} + +static int ipll_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + u32 regval, detected = 0; + unsigned long flags; + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + ipll_find_rate(pll->pll_limit, parent_rate, &rate, &detected); + pll_get_mode_ctrl(PLL_GET_DIV_SEL(detected), + ipll_check_mode_ctrl_restrict, + pll->pll_limit, &detected); + + spin_lock_irqsave(pll->common.lock, flags); + + regval = readl(pll->common.base + pll->pll_reg); + regval = PLL_COPY_REG(regval, detected); + + writel(regval, pll->common.base + pll->pll_reg); + + spin_unlock_irqrestore(pll->common.lock, flags); + + cv1800_clk_wait_for_lock(&pll->common, pll->pll_status.reg, + BIT(pll->pll_status.shift)); + + return 0; +} + +static int pll_enable(struct clk_hw *hw) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + return cv1800_clk_clearbit(&pll->common, &pll->pll_pwd); +} + +static void pll_disable(struct clk_hw *hw) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + cv1800_clk_setbit(&pll->common, &pll->pll_pwd); +} + +static int pll_is_enable(struct clk_hw *hw) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + return cv1800_clk_checkbit(&pll->common, &pll->pll_pwd) == 0; +} + +const struct clk_ops cv1800_clk_ipll_ops = { + .disable = pll_disable, + .enable = pll_enable, + .is_enabled = pll_is_enable, + + .recalc_rate = ipll_recalc_rate, + .determine_rate = ipll_determine_rate, + .set_rate = ipll_set_rate, +}; + +#define PLL_SYN_FACTOR_DOT_POS 26 +#define PLL_SYN_FACTOR_MINIMUM ((4 << PLL_SYN_FACTOR_DOT_POS) + 1) + +static bool fpll_is_factional_mode(struct cv1800_clk_pll *pll) +{ + return cv1800_clk_checkbit(&pll->common, &pll->pll_syn->en); +} + +static unsigned long fpll_calc_rate(unsigned long parent_rate, + unsigned long pre_div_sel, + unsigned long div_sel, + unsigned long post_div_sel, + unsigned long ssc_syn_set, + bool is_full_parent) +{ + u64 dividend = parent_rate * div_sel; + u64 factor = ssc_syn_set * pre_div_sel * post_div_sel; + unsigned long rate; + + dividend <<= PLL_SYN_FACTOR_DOT_POS - 1; + rate = div64_u64_rem(dividend, factor, ÷nd); + + if (is_full_parent) { + dividend <<= 1; + rate <<= 1; + } + + rate += DIV64_U64_ROUND_CLOSEST(dividend, factor); + + return rate; +} + +static unsigned long fpll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + u32 value; + bool clk_full; + u32 syn_set; + + if (!fpll_is_factional_mode(pll)) + return ipll_recalc_rate(hw, parent_rate); + + syn_set = readl(pll->common.base + pll->pll_syn->set); + + if (syn_set == 0) + return 0; + + clk_full = cv1800_clk_checkbit(&pll->common, + &pll->pll_syn->clk_half); + + value = readl(pll->common.base + pll->pll_reg); + + return fpll_calc_rate(parent_rate, + PLL_GET_PRE_DIV_SEL(value), + PLL_GET_DIV_SEL(value), + PLL_GET_POST_DIV_SEL(value), + syn_set, clk_full); +} + +static unsigned long fpll_find_synthesizer(unsigned long parent, + unsigned long rate, + unsigned long pre_div, + unsigned long div, + unsigned long post_div, + bool is_full_parent, + u32 *ssc_syn_set) +{ + u32 test_max = U32_MAX, test_min = PLL_SYN_FACTOR_MINIMUM; + unsigned long trate; + + while (test_min < test_max) { + u32 tssc = (test_max + test_min) / 2; + + trate = fpll_calc_rate(parent, pre_div, div, post_div, + tssc, is_full_parent); + + if (trate == rate) { + test_min = tssc; + break; + } + + if (trate > rate) + test_min = tssc + 1; + else + test_max = tssc - 1; + } + + if (trate != 0) + *ssc_syn_set = test_min; + + return trate; +} + +static int fpll_find_rate(struct cv1800_clk_pll *pll, + const struct cv1800_clk_pll_limit *limit, + unsigned long prate, + unsigned long *rate, + u32 *value, u32 *ssc_syn_set) +{ + unsigned long best_rate = 0; + unsigned long pre_div_sel = 0, div_sel = 0, post_div_sel = 0; + unsigned long pre, div, post; + unsigned long trate = *rate; + u32 detected = *value; + unsigned long tmp; + bool clk_full = cv1800_clk_checkbit(&pll->common, + &pll->pll_syn->clk_half); + + for_each_pll_limit_range(pre, &limit->pre_div) { + for_each_pll_limit_range(post, &limit->post_div) { + for_each_pll_limit_range(div, &limit->div) { + tmp = fpll_find_synthesizer(prate, trate, + pre, div, post, + clk_full, + ssc_syn_set); + + if ((trate - tmp) < (trate - best_rate)) { + best_rate = tmp; + pre_div_sel = pre; + div_sel = div; + post_div_sel = post; + } + } + } + } + + if (best_rate) { + detected = PLL_SET_PRE_DIV_SEL(detected, pre_div_sel); + detected = PLL_SET_POST_DIV_SEL(detected, post_div_sel); + detected = PLL_SET_DIV_SEL(detected, div_sel); + *value = detected; + *rate = best_rate; + return 0; + } + + return -EINVAL; +} + +static int fpll_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + u32 val, ssc_syn_set; + + if (!fpll_is_factional_mode(pll)) + return ipll_determine_rate(hw, req); + + fpll_find_rate(pll, &pll->pll_limit[2], req->best_parent_rate, + &req->rate, &val, &ssc_syn_set); + + return 0; +} + +static bool fpll_check_mode_ctrl_restrict(unsigned long div_sel, + unsigned long ictrl, + unsigned long mode) +{ + unsigned long left_rest = 10 * div_sel; + unsigned long right_rest = 24 * div_sel; + unsigned long test = 184 * (1 + mode) * (1 + ictrl) / 2; + + return test > left_rest && test <= right_rest; +} + +static int fpll_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + u32 regval; + u32 detected = 0, detected_ssc = 0; + unsigned long flags; + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + if (!fpll_is_factional_mode(pll)) + return ipll_set_rate(hw, rate, parent_rate); + + fpll_find_rate(pll, &pll->pll_limit[2], parent_rate, + &rate, &detected, &detected_ssc); + pll_get_mode_ctrl(PLL_GET_DIV_SEL(detected), + fpll_check_mode_ctrl_restrict, + pll->pll_limit, &detected); + + spin_lock_irqsave(pll->common.lock, flags); + + writel(detected_ssc, pll->common.base + pll->pll_syn->set); + + regval = readl(pll->common.base + pll->pll_reg); + regval = PLL_COPY_REG(regval, detected); + + writel(regval, pll->common.base + pll->pll_reg); + + spin_unlock_irqrestore(pll->common.lock, flags); + + cv1800_clk_wait_for_lock(&pll->common, pll->pll_status.reg, + BIT(pll->pll_status.shift)); + + return 0; +} + +static u8 fpll_get_parent(struct clk_hw *hw) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + if (fpll_is_factional_mode(pll)) + return 1; + + return 0; +} + +static int fpll_set_parent(struct clk_hw *hw, u8 index) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + if (index) + cv1800_clk_setbit(&pll->common, &pll->pll_syn->en); + else + cv1800_clk_clearbit(&pll->common, &pll->pll_syn->en); + + return 0; +} + +const struct clk_ops cv1800_clk_fpll_ops = { + .disable = pll_disable, + .enable = pll_enable, + .is_enabled = pll_is_enable, + + .recalc_rate = fpll_recalc_rate, + .determine_rate = fpll_determine_rate, + .set_rate = fpll_set_rate, + + .set_parent = fpll_set_parent, + .get_parent = fpll_get_parent, +}; diff --git a/drivers/clk/sophgo/clk-cv18xx-pll.h b/drivers/clk/sophgo/clk-cv18xx-pll.h new file mode 100644 index 000000000000..7a33f3da2d64 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-pll.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023 Inochi Amaoto <inochiama@outlook.com> + */ + +#ifndef _CLK_SOPHGO_CV1800_PLL_H_ +#define _CLK_SOPHGO_CV1800_PLL_H_ + +#include "clk-cv18xx-common.h" + +struct cv1800_clk_pll_limit { + struct { + u8 min; + u8 max; + } pre_div, div, post_div, ictrl, mode; +}; + +#define _CV1800_PLL_LIMIT(_min, _max) \ + { \ + .min = _min, \ + .max = _max, \ + } \ + +#define for_each_pll_limit_range(_var, _restrict) \ + for (_var = (_restrict)->min; _var <= (_restrict)->max; _var++) + +struct cv1800_clk_pll_synthesizer { + struct cv1800_clk_regbit en; + struct cv1800_clk_regbit clk_half; + u32 ctrl; + u32 set; +}; + +#define _PLL_PRE_DIV_SEL_FIELD GENMASK(6, 0) +#define _PLL_POST_DIV_SEL_FIELD GENMASK(14, 8) +#define _PLL_SEL_MODE_FIELD GENMASK(16, 15) +#define _PLL_DIV_SEL_FIELD GENMASK(23, 17) +#define _PLL_ICTRL_FIELD GENMASK(26, 24) + +#define _PLL_ALL_FIELD_MASK \ + (_PLL_PRE_DIV_SEL_FIELD | \ + _PLL_POST_DIV_SEL_FIELD | \ + _PLL_SEL_MODE_FIELD | \ + _PLL_DIV_SEL_FIELD | \ + _PLL_ICTRL_FIELD) + +#define PLL_COPY_REG(_dest, _src) \ + (((_dest) & (~_PLL_ALL_FIELD_MASK)) | ((_src) & _PLL_ALL_FIELD_MASK)) + +#define PLL_GET_PRE_DIV_SEL(_reg) \ + FIELD_GET(_PLL_PRE_DIV_SEL_FIELD, (_reg)) +#define PLL_GET_POST_DIV_SEL(_reg) \ + FIELD_GET(_PLL_POST_DIV_SEL_FIELD, (_reg)) +#define PLL_GET_SEL_MODE(_reg) \ + FIELD_GET(_PLL_SEL_MODE_FIELD, (_reg)) +#define PLL_GET_DIV_SEL(_reg) \ + FIELD_GET(_PLL_DIV_SEL_FIELD, (_reg)) +#define PLL_GET_ICTRL(_reg) \ + FIELD_GET(_PLL_ICTRL_FIELD, (_reg)) + +#define PLL_SET_PRE_DIV_SEL(_reg, _val) \ + _CV1800_SET_FIELD((_reg), (_val), _PLL_PRE_DIV_SEL_FIELD) +#define PLL_SET_POST_DIV_SEL(_reg, _val) \ + _CV1800_SET_FIELD((_reg), (_val), _PLL_POST_DIV_SEL_FIELD) +#define PLL_SET_SEL_MODE(_reg, _val) \ + _CV1800_SET_FIELD((_reg), (_val), _PLL_SEL_MODE_FIELD) +#define PLL_SET_DIV_SEL(_reg, _val) \ + _CV1800_SET_FIELD((_reg), (_val), _PLL_DIV_SEL_FIELD) +#define PLL_SET_ICTRL(_reg, _val) \ + _CV1800_SET_FIELD((_reg), (_val), _PLL_ICTRL_FIELD) + +struct cv1800_clk_pll { + struct cv1800_clk_common common; + u32 pll_reg; + struct cv1800_clk_regbit pll_pwd; + struct cv1800_clk_regbit pll_status; + const struct cv1800_clk_pll_limit *pll_limit; + struct cv1800_clk_pll_synthesizer *pll_syn; +}; + +#define CV1800_INTEGRAL_PLL(_name, _parent, _pll_reg, \ + _pll_pwd_reg, _pll_pwd_shift, \ + _pll_status_reg, _pll_status_shift, \ + _pll_limit, _flags) \ + struct cv1800_clk_pll _name = { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + &cv1800_clk_ipll_ops,\ + _flags), \ + .pll_reg = _pll_reg, \ + .pll_pwd = CV1800_CLK_BIT(_pll_pwd_reg, \ + _pll_pwd_shift), \ + .pll_status = CV1800_CLK_BIT(_pll_status_reg, \ + _pll_status_shift), \ + .pll_limit = _pll_limit, \ + .pll_syn = NULL, \ + } + +#define CV1800_FACTIONAL_PLL(_name, _parent, _pll_reg, \ + _pll_pwd_reg, _pll_pwd_shift, \ + _pll_status_reg, _pll_status_shift, \ + _pll_limit, _pll_syn, _flags) \ + struct cv1800_clk_pll _name = { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + &cv1800_clk_fpll_ops,\ + _flags), \ + .pll_reg = _pll_reg, \ + .pll_pwd = CV1800_CLK_BIT(_pll_pwd_reg, \ + _pll_pwd_shift), \ + .pll_status = CV1800_CLK_BIT(_pll_status_reg, \ + _pll_status_shift), \ + .pll_limit = _pll_limit, \ + .pll_syn = _pll_syn, \ + } + +extern const struct clk_ops cv1800_clk_ipll_ops; +extern const struct clk_ops cv1800_clk_fpll_ops; + +#endif // _CLK_SOPHGO_CV1800_PLL_H_ diff --git a/drivers/clk/stm32/Kconfig b/drivers/clk/stm32/Kconfig index 3c8493a94a11..dca409d52652 100644 --- a/drivers/clk/stm32/Kconfig +++ b/drivers/clk/stm32/Kconfig @@ -25,5 +25,12 @@ config COMMON_CLK_STM32MP157 help Support for stm32mp15x SoC family clocks. +config COMMON_CLK_STM32MP257 + bool "Clock driver for stm32mp25x clocks" + depends on ARM64 || COMPILE_TEST + default y + help + Support for stm32mp25x SoC family clocks. + endif diff --git a/drivers/clk/stm32/Makefile b/drivers/clk/stm32/Makefile index 5ced7fe3ddec..0a627164fcce 100644 --- a/drivers/clk/stm32/Makefile +++ b/drivers/clk/stm32/Makefile @@ -1,2 +1,3 @@ obj-$(CONFIG_COMMON_CLK_STM32MP135) += clk-stm32mp13.o clk-stm32-core.o reset-stm32.o obj-$(CONFIG_COMMON_CLK_STM32MP157) += clk-stm32mp1.o reset-stm32.o +obj-$(CONFIG_COMMON_CLK_STM32MP257) += clk-stm32mp25.o clk-stm32-core.o reset-stm32.o diff --git a/drivers/clk/stm32/clk-stm32-core.c b/drivers/clk/stm32/clk-stm32-core.c index 58705fcad334..1721a3ed7386 100644 --- a/drivers/clk/stm32/clk-stm32-core.c +++ b/drivers/clk/stm32/clk-stm32-core.c @@ -25,7 +25,6 @@ static int stm32_rcc_clock_init(struct device *dev, { const struct stm32_rcc_match_data *data = match->data; struct clk_hw_onecell_data *clk_data = data->hw_clks; - struct device_node *np = dev_of_node(dev); struct clk_hw **hws; int n, max_binding; @@ -64,7 +63,7 @@ static int stm32_rcc_clock_init(struct device *dev, hws[cfg_clock->id] = hw; } - return of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data); } int stm32_rcc_init(struct device *dev, const struct of_device_id *match_data, @@ -638,7 +637,7 @@ struct clk_hw *clk_stm32_mux_register(struct device *dev, mux->lock = lock; mux->clock_data = data->clock_data; - err = clk_hw_register(dev, hw); + err = devm_clk_hw_register(dev, hw); if (err) return ERR_PTR(err); @@ -659,7 +658,7 @@ struct clk_hw *clk_stm32_gate_register(struct device *dev, gate->lock = lock; gate->clock_data = data->clock_data; - err = clk_hw_register(dev, hw); + err = devm_clk_hw_register(dev, hw); if (err) return ERR_PTR(err); @@ -680,7 +679,7 @@ struct clk_hw *clk_stm32_div_register(struct device *dev, div->lock = lock; div->clock_data = data->clock_data; - err = clk_hw_register(dev, hw); + err = devm_clk_hw_register(dev, hw); if (err) return ERR_PTR(err); @@ -701,7 +700,7 @@ struct clk_hw *clk_stm32_composite_register(struct device *dev, composite->lock = lock; composite->clock_data = data->clock_data; - err = clk_hw_register(dev, hw); + err = devm_clk_hw_register(dev, hw); if (err) return ERR_PTR(err); diff --git a/drivers/clk/stm32/clk-stm32mp13.c b/drivers/clk/stm32/clk-stm32mp13.c index d4ecb3c34a1b..bf81d7491708 100644 --- a/drivers/clk/stm32/clk-stm32mp13.c +++ b/drivers/clk/stm32/clk-stm32mp13.c @@ -1536,77 +1536,16 @@ static const struct of_device_id stm32mp13_match_data[] = { }; MODULE_DEVICE_TABLE(of, stm32mp13_match_data); -static int stm32mp1_rcc_init(struct device *dev) -{ - void __iomem *rcc_base; - int ret = -ENOMEM; - - rcc_base = of_iomap(dev_of_node(dev), 0); - if (!rcc_base) { - dev_err(dev, "%pOFn: unable to map resource", dev_of_node(dev)); - goto out; - } - - ret = stm32_rcc_init(dev, stm32mp13_match_data, rcc_base); -out: - if (ret) { - if (rcc_base) - iounmap(rcc_base); - - of_node_put(dev_of_node(dev)); - } - - return ret; -} - -static int get_clock_deps(struct device *dev) -{ - static const char * const clock_deps_name[] = { - "hsi", "hse", "csi", "lsi", "lse", - }; - size_t deps_size = sizeof(struct clk *) * ARRAY_SIZE(clock_deps_name); - struct clk **clk_deps; - int i; - - clk_deps = devm_kzalloc(dev, deps_size, GFP_KERNEL); - if (!clk_deps) - return -ENOMEM; - - for (i = 0; i < ARRAY_SIZE(clock_deps_name); i++) { - struct clk *clk = of_clk_get_by_name(dev_of_node(dev), - clock_deps_name[i]); - - if (IS_ERR(clk)) { - if (PTR_ERR(clk) != -EINVAL && PTR_ERR(clk) != -ENOENT) - return PTR_ERR(clk); - } else { - /* Device gets a reference count on the clock */ - clk_deps[i] = devm_clk_get(dev, __clk_get_name(clk)); - clk_put(clk); - } - } - - return 0; -} - static int stm32mp1_rcc_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - int ret = get_clock_deps(dev); + void __iomem *base; - if (!ret) - ret = stm32mp1_rcc_init(dev); - - return ret; -} - -static void stm32mp1_rcc_clocks_remove(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct device_node *child, *np = dev_of_node(dev); + base = devm_platform_ioremap_resource(pdev, 0); + if (WARN_ON(IS_ERR(base))) + return PTR_ERR(base); - for_each_available_child_of_node(np, child) - of_clk_del_provider(child); + return stm32_rcc_init(dev, stm32mp13_match_data, base); } static struct platform_driver stm32mp13_rcc_clocks_driver = { @@ -1615,7 +1554,6 @@ static struct platform_driver stm32mp13_rcc_clocks_driver = { .of_match_table = stm32mp13_match_data, }, .probe = stm32mp1_rcc_clocks_probe, - .remove_new = stm32mp1_rcc_clocks_remove, }; static int __init stm32mp13_clocks_init(void) diff --git a/drivers/clk/stm32/clk-stm32mp25.c b/drivers/clk/stm32/clk-stm32mp25.c new file mode 100644 index 000000000000..210b75b39e50 --- /dev/null +++ b/drivers/clk/stm32/clk-stm32mp25.c @@ -0,0 +1,1875 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) STMicroelectronics 2023 - All Rights Reserved + * Author: Gabriel Fernandez <gabriel.fernandez@foss.st.com> for STMicroelectronics. + */ + +#include <linux/clk-provider.h> +#include <linux/platform_device.h> + +#include "clk-stm32-core.h" +#include "reset-stm32.h" +#include "stm32mp25_rcc.h" + +#include <dt-bindings/clock/st,stm32mp25-rcc.h> +#include <dt-bindings/reset/st,stm32mp25-rcc.h> + +enum { + HSE, + HSI, + MSI, + LSE, + LSI, + HSE_DIV2, + ICN_HS_MCU, + ICN_LS_MCU, + ICN_SDMMC, + ICN_DDR, + ICN_DISPLAY, + ICN_HSL, + ICN_NIC, + ICN_VID, + FLEXGEN_07, + FLEXGEN_08, + FLEXGEN_09, + FLEXGEN_10, + FLEXGEN_11, + FLEXGEN_12, + FLEXGEN_13, + FLEXGEN_14, + FLEXGEN_15, + FLEXGEN_16, + FLEXGEN_17, + FLEXGEN_18, + FLEXGEN_19, + FLEXGEN_20, + FLEXGEN_21, + FLEXGEN_22, + FLEXGEN_23, + FLEXGEN_24, + FLEXGEN_25, + FLEXGEN_26, + FLEXGEN_27, + FLEXGEN_28, + FLEXGEN_29, + FLEXGEN_30, + FLEXGEN_31, + FLEXGEN_32, + FLEXGEN_33, + FLEXGEN_34, + FLEXGEN_35, + FLEXGEN_36, + FLEXGEN_37, + FLEXGEN_38, + FLEXGEN_39, + FLEXGEN_40, + FLEXGEN_41, + FLEXGEN_42, + FLEXGEN_43, + FLEXGEN_44, + FLEXGEN_45, + FLEXGEN_46, + FLEXGEN_47, + FLEXGEN_48, + FLEXGEN_49, + FLEXGEN_50, + FLEXGEN_51, + FLEXGEN_52, + FLEXGEN_53, + FLEXGEN_54, + FLEXGEN_55, + FLEXGEN_56, + FLEXGEN_57, + FLEXGEN_58, + FLEXGEN_59, + FLEXGEN_60, + FLEXGEN_61, + FLEXGEN_62, + FLEXGEN_63, + ICN_APB1, + ICN_APB2, + ICN_APB3, + ICN_APB4, + ICN_APBDBG, + TIMG1, + TIMG2, + PLL3, + DSI_TXBYTE, +}; + +static const struct clk_parent_data adc12_src[] = { + { .index = FLEXGEN_46 }, + { .index = ICN_LS_MCU }, +}; + +static const struct clk_parent_data adc3_src[] = { + { .index = FLEXGEN_47 }, + { .index = ICN_LS_MCU }, + { .index = FLEXGEN_46 }, +}; + +static const struct clk_parent_data usb2phy1_src[] = { + { .index = FLEXGEN_57 }, + { .index = HSE_DIV2 }, +}; + +static const struct clk_parent_data usb2phy2_src[] = { + { .index = FLEXGEN_58 }, + { .index = HSE_DIV2 }, +}; + +static const struct clk_parent_data usb3pciphy_src[] = { + { .index = FLEXGEN_34 }, + { .index = HSE_DIV2 }, +}; + +static struct clk_stm32_gate ck_ker_ltdc; + +static const struct clk_parent_data dsiblane_src[] = { + { .index = DSI_TXBYTE }, + { .hw = &ck_ker_ltdc.hw }, +}; + +static const struct clk_parent_data dsiphy_src[] = { + { .index = FLEXGEN_28 }, + { .index = HSE }, +}; + +static const struct clk_parent_data lvdsphy_src[] = { + { .index = FLEXGEN_32 }, + { .index = HSE }, +}; + +static const struct clk_parent_data dts_src[] = { + { .index = HSI }, + { .index = HSE }, + { .index = MSI }, +}; + +static const struct clk_parent_data mco1_src[] = { + { .index = FLEXGEN_61 }, +}; + +static const struct clk_parent_data mco2_src[] = { + { .index = FLEXGEN_62 }, +}; + +enum enum_mux_cfg { + MUX_ADC12, + MUX_ADC3, + MUX_DSIBLANE, + MUX_DSIPHY, + MUX_DTS, + MUX_LVDSPHY, + MUX_MCO1, + MUX_MCO2, + MUX_USB2PHY1, + MUX_USB2PHY2, + MUX_USB3PCIEPHY, + MUX_NB +}; + +#define MUX_CFG(id, _offset, _shift, _witdh) \ + [id] = { \ + .offset = (_offset), \ + .shift = (_shift), \ + .width = (_witdh), \ + } + +static const struct stm32_mux_cfg stm32mp25_muxes[MUX_NB] = { + MUX_CFG(MUX_ADC12, RCC_ADC12CFGR, 12, 1), + MUX_CFG(MUX_ADC3, RCC_ADC3CFGR, 12, 2), + MUX_CFG(MUX_DSIBLANE, RCC_DSICFGR, 12, 1), + MUX_CFG(MUX_DSIPHY, RCC_DSICFGR, 15, 1), + MUX_CFG(MUX_DTS, RCC_DTSCFGR, 12, 2), + MUX_CFG(MUX_LVDSPHY, RCC_LVDSCFGR, 15, 1), + MUX_CFG(MUX_MCO1, RCC_MCO1CFGR, 0, 1), + MUX_CFG(MUX_MCO2, RCC_MCO2CFGR, 0, 1), + MUX_CFG(MUX_USB2PHY1, RCC_USB2PHY1CFGR, 15, 1), + MUX_CFG(MUX_USB2PHY2, RCC_USB2PHY2CFGR, 15, 1), + MUX_CFG(MUX_USB3PCIEPHY, RCC_USB3PCIEPHYCFGR, 15, 1), +}; + +enum enum_gate_cfg { + GATE_ADC12, + GATE_ADC3, + GATE_ADF1, + GATE_CCI, + GATE_CRC, + GATE_CRYP1, + GATE_CRYP2, + GATE_CSI, + GATE_DCMIPP, + GATE_DSI, + GATE_DTS, + GATE_ETH1, + GATE_ETH1MAC, + GATE_ETH1RX, + GATE_ETH1STP, + GATE_ETH1TX, + GATE_ETH2, + GATE_ETH2MAC, + GATE_ETH2RX, + GATE_ETH2STP, + GATE_ETH2TX, + GATE_ETHSW, + GATE_ETHSWACMCFG, + GATE_ETHSWACMMSG, + GATE_ETHSWMAC, + GATE_ETHSWREF, + GATE_FDCAN, + GATE_GPU, + GATE_HASH, + GATE_HDP, + GATE_I2C1, + GATE_I2C2, + GATE_I2C3, + GATE_I2C4, + GATE_I2C5, + GATE_I2C6, + GATE_I2C7, + GATE_I2C8, + GATE_I3C1, + GATE_I3C2, + GATE_I3C3, + GATE_I3C4, + GATE_IS2M, + GATE_IWDG1, + GATE_IWDG2, + GATE_IWDG3, + GATE_IWDG4, + GATE_IWDG5, + GATE_LPTIM1, + GATE_LPTIM2, + GATE_LPTIM3, + GATE_LPTIM4, + GATE_LPTIM5, + GATE_LPUART1, + GATE_LTDC, + GATE_LVDS, + GATE_MCO1, + GATE_MCO2, + GATE_MDF1, + GATE_OSPIIOM, + GATE_PCIE, + GATE_PKA, + GATE_RNG, + GATE_SAES, + GATE_SAI1, + GATE_SAI2, + GATE_SAI3, + GATE_SAI4, + GATE_SDMMC1, + GATE_SDMMC2, + GATE_SDMMC3, + GATE_SERC, + GATE_SPDIFRX, + GATE_SPI1, + GATE_SPI2, + GATE_SPI3, + GATE_SPI4, + GATE_SPI5, + GATE_SPI6, + GATE_SPI7, + GATE_SPI8, + GATE_TIM1, + GATE_TIM10, + GATE_TIM11, + GATE_TIM12, + GATE_TIM13, + GATE_TIM14, + GATE_TIM15, + GATE_TIM16, + GATE_TIM17, + GATE_TIM2, + GATE_TIM20, + GATE_TIM3, + GATE_TIM4, + GATE_TIM5, + GATE_TIM6, + GATE_TIM7, + GATE_TIM8, + GATE_UART4, + GATE_UART5, + GATE_UART7, + GATE_UART8, + GATE_UART9, + GATE_USART1, + GATE_USART2, + GATE_USART3, + GATE_USART6, + GATE_USBH, + GATE_USB2PHY1, + GATE_USB2PHY2, + GATE_USB3DR, + GATE_USB3PCIEPHY, + GATE_USBTC, + GATE_VDEC, + GATE_VENC, + GATE_VREF, + GATE_WWDG1, + GATE_WWDG2, + GATE_NB +}; + +#define GATE_CFG(id, _offset, _bit_idx, _offset_clr) \ + [id] = { \ + .offset = (_offset), \ + .bit_idx = (_bit_idx), \ + .set_clr = (_offset_clr), \ + } + +static const struct stm32_gate_cfg stm32mp25_gates[GATE_NB] = { + GATE_CFG(GATE_ADC12, RCC_ADC12CFGR, 1, 0), + GATE_CFG(GATE_ADC3, RCC_ADC3CFGR, 1, 0), + GATE_CFG(GATE_ADF1, RCC_ADF1CFGR, 1, 0), + GATE_CFG(GATE_CCI, RCC_CCICFGR, 1, 0), + GATE_CFG(GATE_CRC, RCC_CRCCFGR, 1, 0), + GATE_CFG(GATE_CRYP1, RCC_CRYP1CFGR, 1, 0), + GATE_CFG(GATE_CRYP2, RCC_CRYP2CFGR, 1, 0), + GATE_CFG(GATE_CSI, RCC_CSICFGR, 1, 0), + GATE_CFG(GATE_DCMIPP, RCC_DCMIPPCFGR, 1, 0), + GATE_CFG(GATE_DSI, RCC_DSICFGR, 1, 0), + GATE_CFG(GATE_DTS, RCC_DTSCFGR, 1, 0), + GATE_CFG(GATE_ETH1, RCC_ETH1CFGR, 5, 0), + GATE_CFG(GATE_ETH1MAC, RCC_ETH1CFGR, 1, 0), + GATE_CFG(GATE_ETH1RX, RCC_ETH1CFGR, 10, 0), + GATE_CFG(GATE_ETH1STP, RCC_ETH1CFGR, 4, 0), + GATE_CFG(GATE_ETH1TX, RCC_ETH1CFGR, 8, 0), + GATE_CFG(GATE_ETH2, RCC_ETH2CFGR, 5, 0), + GATE_CFG(GATE_ETH2MAC, RCC_ETH2CFGR, 1, 0), + GATE_CFG(GATE_ETH2RX, RCC_ETH2CFGR, 10, 0), + GATE_CFG(GATE_ETH2STP, RCC_ETH2CFGR, 4, 0), + GATE_CFG(GATE_ETH2TX, RCC_ETH2CFGR, 8, 0), + GATE_CFG(GATE_ETHSW, RCC_ETHSWCFGR, 5, 0), + GATE_CFG(GATE_ETHSWACMCFG, RCC_ETHSWACMCFGR, 1, 0), + GATE_CFG(GATE_ETHSWACMMSG, RCC_ETHSWACMMSGCFGR, 1, 0), + GATE_CFG(GATE_ETHSWMAC, RCC_ETHSWCFGR, 1, 0), + GATE_CFG(GATE_ETHSWREF, RCC_ETHSWCFGR, 21, 0), + GATE_CFG(GATE_FDCAN, RCC_FDCANCFGR, 1, 0), + GATE_CFG(GATE_GPU, RCC_GPUCFGR, 1, 0), + GATE_CFG(GATE_HASH, RCC_HASHCFGR, 1, 0), + GATE_CFG(GATE_HDP, RCC_HDPCFGR, 1, 0), + GATE_CFG(GATE_I2C1, RCC_I2C1CFGR, 1, 0), + GATE_CFG(GATE_I2C2, RCC_I2C2CFGR, 1, 0), + GATE_CFG(GATE_I2C3, RCC_I2C3CFGR, 1, 0), + GATE_CFG(GATE_I2C4, RCC_I2C4CFGR, 1, 0), + GATE_CFG(GATE_I2C5, RCC_I2C5CFGR, 1, 0), + GATE_CFG(GATE_I2C6, RCC_I2C6CFGR, 1, 0), + GATE_CFG(GATE_I2C7, RCC_I2C7CFGR, 1, 0), + GATE_CFG(GATE_I2C8, RCC_I2C8CFGR, 1, 0), + GATE_CFG(GATE_I3C1, RCC_I3C1CFGR, 1, 0), + GATE_CFG(GATE_I3C2, RCC_I3C2CFGR, 1, 0), + GATE_CFG(GATE_I3C3, RCC_I3C3CFGR, 1, 0), + GATE_CFG(GATE_I3C4, RCC_I3C4CFGR, 1, 0), + GATE_CFG(GATE_IS2M, RCC_IS2MCFGR, 1, 0), + GATE_CFG(GATE_IWDG1, RCC_IWDG1CFGR, 1, 0), + GATE_CFG(GATE_IWDG2, RCC_IWDG2CFGR, 1, 0), + GATE_CFG(GATE_IWDG3, RCC_IWDG3CFGR, 1, 0), + GATE_CFG(GATE_IWDG4, RCC_IWDG4CFGR, 1, 0), + GATE_CFG(GATE_IWDG5, RCC_IWDG5CFGR, 1, 0), + GATE_CFG(GATE_LPTIM1, RCC_LPTIM1CFGR, 1, 0), + GATE_CFG(GATE_LPTIM2, RCC_LPTIM2CFGR, 1, 0), + GATE_CFG(GATE_LPTIM3, RCC_LPTIM3CFGR, 1, 0), + GATE_CFG(GATE_LPTIM4, RCC_LPTIM4CFGR, 1, 0), + GATE_CFG(GATE_LPTIM5, RCC_LPTIM5CFGR, 1, 0), + GATE_CFG(GATE_LPUART1, RCC_LPUART1CFGR, 1, 0), + GATE_CFG(GATE_LTDC, RCC_LTDCCFGR, 1, 0), + GATE_CFG(GATE_LVDS, RCC_LVDSCFGR, 1, 0), + GATE_CFG(GATE_MCO1, RCC_MCO1CFGR, 8, 0), + GATE_CFG(GATE_MCO2, RCC_MCO2CFGR, 8, 0), + GATE_CFG(GATE_MDF1, RCC_MDF1CFGR, 1, 0), + GATE_CFG(GATE_OSPIIOM, RCC_OSPIIOMCFGR, 1, 0), + GATE_CFG(GATE_PCIE, RCC_PCIECFGR, 1, 0), + GATE_CFG(GATE_PKA, RCC_PKACFGR, 1, 0), + GATE_CFG(GATE_RNG, RCC_RNGCFGR, 1, 0), + GATE_CFG(GATE_SAES, RCC_SAESCFGR, 1, 0), + GATE_CFG(GATE_SAI1, RCC_SAI1CFGR, 1, 0), + GATE_CFG(GATE_SAI2, RCC_SAI2CFGR, 1, 0), + GATE_CFG(GATE_SAI3, RCC_SAI3CFGR, 1, 0), + GATE_CFG(GATE_SAI4, RCC_SAI4CFGR, 1, 0), + GATE_CFG(GATE_SDMMC1, RCC_SDMMC1CFGR, 1, 0), + GATE_CFG(GATE_SDMMC2, RCC_SDMMC2CFGR, 1, 0), + GATE_CFG(GATE_SDMMC3, RCC_SDMMC3CFGR, 1, 0), + GATE_CFG(GATE_SERC, RCC_SERCCFGR, 1, 0), + GATE_CFG(GATE_SPDIFRX, RCC_SPDIFRXCFGR, 1, 0), + GATE_CFG(GATE_SPI1, RCC_SPI1CFGR, 1, 0), + GATE_CFG(GATE_SPI2, RCC_SPI2CFGR, 1, 0), + GATE_CFG(GATE_SPI3, RCC_SPI3CFGR, 1, 0), + GATE_CFG(GATE_SPI4, RCC_SPI4CFGR, 1, 0), + GATE_CFG(GATE_SPI5, RCC_SPI5CFGR, 1, 0), + GATE_CFG(GATE_SPI6, RCC_SPI6CFGR, 1, 0), + GATE_CFG(GATE_SPI7, RCC_SPI7CFGR, 1, 0), + GATE_CFG(GATE_SPI8, RCC_SPI8CFGR, 1, 0), + GATE_CFG(GATE_TIM1, RCC_TIM1CFGR, 1, 0), + GATE_CFG(GATE_TIM10, RCC_TIM10CFGR, 1, 0), + GATE_CFG(GATE_TIM11, RCC_TIM11CFGR, 1, 0), + GATE_CFG(GATE_TIM12, RCC_TIM12CFGR, 1, 0), + GATE_CFG(GATE_TIM13, RCC_TIM13CFGR, 1, 0), + GATE_CFG(GATE_TIM14, RCC_TIM14CFGR, 1, 0), + GATE_CFG(GATE_TIM15, RCC_TIM15CFGR, 1, 0), + GATE_CFG(GATE_TIM16, RCC_TIM16CFGR, 1, 0), + GATE_CFG(GATE_TIM17, RCC_TIM17CFGR, 1, 0), + GATE_CFG(GATE_TIM2, RCC_TIM2CFGR, 1, 0), + GATE_CFG(GATE_TIM20, RCC_TIM20CFGR, 1, 0), + GATE_CFG(GATE_TIM3, RCC_TIM3CFGR, 1, 0), + GATE_CFG(GATE_TIM4, RCC_TIM4CFGR, 1, 0), + GATE_CFG(GATE_TIM5, RCC_TIM5CFGR, 1, 0), + GATE_CFG(GATE_TIM6, RCC_TIM6CFGR, 1, 0), + GATE_CFG(GATE_TIM7, RCC_TIM7CFGR, 1, 0), + GATE_CFG(GATE_TIM8, RCC_TIM8CFGR, 1, 0), + GATE_CFG(GATE_UART4, RCC_UART4CFGR, 1, 0), + GATE_CFG(GATE_UART5, RCC_UART5CFGR, 1, 0), + GATE_CFG(GATE_UART7, RCC_UART7CFGR, 1, 0), + GATE_CFG(GATE_UART8, RCC_UART8CFGR, 1, 0), + GATE_CFG(GATE_UART9, RCC_UART9CFGR, 1, 0), + GATE_CFG(GATE_USART1, RCC_USART1CFGR, 1, 0), + GATE_CFG(GATE_USART2, RCC_USART2CFGR, 1, 0), + GATE_CFG(GATE_USART3, RCC_USART3CFGR, 1, 0), + GATE_CFG(GATE_USART6, RCC_USART6CFGR, 1, 0), + GATE_CFG(GATE_USBH, RCC_USBHCFGR, 1, 0), + GATE_CFG(GATE_USB2PHY1, RCC_USB2PHY1CFGR, 1, 0), + GATE_CFG(GATE_USB2PHY2, RCC_USB2PHY2CFGR, 1, 0), + GATE_CFG(GATE_USB3DR, RCC_USB3DRCFGR, 1, 0), + GATE_CFG(GATE_USB3PCIEPHY, RCC_USB3PCIEPHYCFGR, 1, 0), + GATE_CFG(GATE_USBTC, RCC_USBTCCFGR, 1, 0), + GATE_CFG(GATE_VDEC, RCC_VDECCFGR, 1, 0), + GATE_CFG(GATE_VENC, RCC_VENCCFGR, 1, 0), + GATE_CFG(GATE_VREF, RCC_VREFCFGR, 1, 0), + GATE_CFG(GATE_WWDG1, RCC_WWDG1CFGR, 1, 0), + GATE_CFG(GATE_WWDG2, RCC_WWDG2CFGR, 1, 0), +}; + +#define CLK_HW_INIT_INDEX(_name, _parent, _ops, _flags) \ + (&(struct clk_init_data) { \ + .flags = _flags, \ + .name = _name, \ + .parent_data = (const struct clk_parent_data[]) { \ + { .index = _parent }, \ + }, \ + .num_parents = 1, \ + .ops = _ops, \ + }) + +/* ADC */ +static struct clk_stm32_gate ck_icn_p_adc12 = { + .gate_id = GATE_ADC12, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_adc12", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_composite ck_ker_adc12 = { + .gate_id = GATE_ADC12, + .mux_id = MUX_ADC12, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_adc12", adc12_src, &clk_stm32_composite_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_adc3 = { + .gate_id = GATE_ADC3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_adc3", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_composite ck_ker_adc3 = { + .gate_id = GATE_ADC3, + .mux_id = MUX_ADC3, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_adc3", adc3_src, &clk_stm32_composite_ops, 0), +}; + +/* ADF */ +static struct clk_stm32_gate ck_icn_p_adf1 = { + .gate_id = GATE_ADF1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_adf1", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_adf1 = { + .gate_id = GATE_ADF1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_adf1", FLEXGEN_42, &clk_stm32_gate_ops, 0), +}; + +/* DCMI */ +static struct clk_stm32_gate ck_icn_p_cci = { + .gate_id = GATE_CCI, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_cci", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* CSI-HOST */ +static struct clk_stm32_gate ck_icn_p_csi = { + .gate_id = GATE_CSI, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_csi", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_csi = { + .gate_id = GATE_CSI, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_csi", FLEXGEN_29, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_csitxesc = { + .gate_id = GATE_CSI, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_csitxesc", FLEXGEN_30, &clk_stm32_gate_ops, 0), +}; + +/* CSI-PHY */ +static struct clk_stm32_gate ck_ker_csiphy = { + .gate_id = GATE_CSI, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_csiphy", FLEXGEN_31, &clk_stm32_gate_ops, 0), +}; + +/* DCMIPP */ +static struct clk_stm32_gate ck_icn_p_dcmipp = { + .gate_id = GATE_DCMIPP, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_dcmipp", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +/* CRC */ +static struct clk_stm32_gate ck_icn_p_crc = { + .gate_id = GATE_CRC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_crc", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* CRYP */ +static struct clk_stm32_gate ck_icn_p_cryp1 = { + .gate_id = GATE_CRYP1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_cryp1", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_cryp2 = { + .gate_id = GATE_CRYP2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_cryp2", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* DBG & TRACE*/ +/* Trace and debug clocks are managed by SCMI */ + +/* LTDC */ +static struct clk_stm32_gate ck_icn_p_ltdc = { + .gate_id = GATE_LTDC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_ltdc", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_ltdc = { + .gate_id = GATE_LTDC, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_ltdc", FLEXGEN_27, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +/* DSI */ +static struct clk_stm32_gate ck_icn_p_dsi = { + .gate_id = GATE_DSI, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_dsi", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_composite clk_lanebyte = { + .gate_id = GATE_DSI, + .mux_id = MUX_DSIBLANE, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("clk_lanebyte", dsiblane_src, + &clk_stm32_composite_ops, 0), +}; + +/* LVDS */ +static struct clk_stm32_gate ck_icn_p_lvds = { + .gate_id = GATE_LVDS, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lvds", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +/* DSI PHY */ +static struct clk_stm32_composite clk_phy_dsi = { + .gate_id = GATE_DSI, + .mux_id = MUX_DSIPHY, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("clk_phy_dsi", dsiphy_src, + &clk_stm32_composite_ops, 0), +}; + +/* LVDS PHY */ +static struct clk_stm32_composite ck_ker_lvdsphy = { + .gate_id = GATE_LVDS, + .mux_id = MUX_LVDSPHY, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_lvdsphy", lvdsphy_src, + &clk_stm32_composite_ops, 0), +}; + +/* DTS */ +static struct clk_stm32_composite ck_ker_dts = { + .gate_id = GATE_DTS, + .mux_id = MUX_DTS, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_dts", dts_src, + &clk_stm32_composite_ops, 0), +}; + +/* ETHERNET */ +static struct clk_stm32_gate ck_icn_p_eth1 = { + .gate_id = GATE_ETH1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_eth1", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1stp = { + .gate_id = GATE_ETH1STP, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1stp", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1 = { + .gate_id = GATE_ETH1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1", FLEXGEN_54, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1ptp = { + .gate_id = GATE_ETH1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1ptp", FLEXGEN_56, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1mac = { + .gate_id = GATE_ETH1MAC, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1mac", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1tx = { + .gate_id = GATE_ETH1TX, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1tx", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1rx = { + .gate_id = GATE_ETH1RX, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1rx", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_eth2 = { + .gate_id = GATE_ETH2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_eth2", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2stp = { + .gate_id = GATE_ETH2STP, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2stp", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2 = { + .gate_id = GATE_ETH2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2", FLEXGEN_55, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2ptp = { + .gate_id = GATE_ETH2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2ptp", FLEXGEN_56, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2mac = { + .gate_id = GATE_ETH2MAC, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2mac", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2tx = { + .gate_id = GATE_ETH2TX, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2tx", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2rx = { + .gate_id = GATE_ETH2RX, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2rx", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_ethsw = { + .gate_id = GATE_ETHSWMAC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_ethsw", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_ethsw = { + .gate_id = GATE_ETHSW, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_ethsw", FLEXGEN_54, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_ethswref = { + .gate_id = GATE_ETHSWREF, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_ethswref", FLEXGEN_60, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_ethsw_acm_cfg = { + .gate_id = GATE_ETHSWACMCFG, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_ethsw_acm_cfg", ICN_LS_MCU, + &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_ethsw_acm_msg = { + .gate_id = GATE_ETHSWACMMSG, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_ethsw_acm_msg", ICN_LS_MCU, + &clk_stm32_gate_ops, 0), +}; + +/* FDCAN */ +static struct clk_stm32_gate ck_icn_p_fdcan = { + .gate_id = GATE_FDCAN, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_fdcan", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_fdcan = { + .gate_id = GATE_FDCAN, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_fdcan", FLEXGEN_26, &clk_stm32_gate_ops, 0), +}; + +/* GPU */ +static struct clk_stm32_gate ck_icn_m_gpu = { + .gate_id = GATE_GPU, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_gpu", FLEXGEN_59, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_gpu = { + .gate_id = GATE_GPU, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_gpu", PLL3, &clk_stm32_gate_ops, 0), +}; + +/* HASH */ +static struct clk_stm32_gate ck_icn_p_hash = { + .gate_id = GATE_HASH, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_hash", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* HDP */ +static struct clk_stm32_gate ck_icn_p_hdp = { + .gate_id = GATE_HDP, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_hdp", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +/* I2C */ +static struct clk_stm32_gate ck_icn_p_i2c8 = { + .gate_id = GATE_I2C8, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c8", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c1 = { + .gate_id = GATE_I2C1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c1", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c2 = { + .gate_id = GATE_I2C2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c3 = { + .gate_id = GATE_I2C3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c3", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c4 = { + .gate_id = GATE_I2C4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c4", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c5 = { + .gate_id = GATE_I2C5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c5", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c6 = { + .gate_id = GATE_I2C6, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c6", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c7 = { + .gate_id = GATE_I2C7, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c7", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c1 = { + .gate_id = GATE_I2C1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c1", FLEXGEN_12, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c2 = { + .gate_id = GATE_I2C2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c2", FLEXGEN_12, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c3 = { + .gate_id = GATE_I2C3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c3", FLEXGEN_13, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c5 = { + .gate_id = GATE_I2C5, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c5", FLEXGEN_13, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c4 = { + .gate_id = GATE_I2C4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c4", FLEXGEN_14, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c6 = { + .gate_id = GATE_I2C6, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c6", FLEXGEN_14, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c7 = { + .gate_id = GATE_I2C7, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c7", FLEXGEN_15, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c8 = { + .gate_id = GATE_I2C8, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c8", FLEXGEN_38, &clk_stm32_gate_ops, 0), +}; + +/* I3C */ +static struct clk_stm32_gate ck_icn_p_i3c1 = { + .gate_id = GATE_I3C1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i3c1", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i3c2 = { + .gate_id = GATE_I3C2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i3c2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i3c3 = { + .gate_id = GATE_I3C3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i3c3", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i3c4 = { + .gate_id = GATE_I3C4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i3c4", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i3c1 = { + .gate_id = GATE_I3C1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i3c1", FLEXGEN_12, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i3c2 = { + .gate_id = GATE_I3C2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i3c2", FLEXGEN_12, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i3c3 = { + .gate_id = GATE_I3C3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i3c3", FLEXGEN_13, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i3c4 = { + .gate_id = GATE_I3C4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i3c4", FLEXGEN_36, &clk_stm32_gate_ops, 0), +}; + +/* I2S */ +static struct clk_stm32_gate ck_icn_p_is2m = { + .gate_id = GATE_IS2M, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_is2m", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +/* IWDG */ +static struct clk_stm32_gate ck_icn_p_iwdg2 = { + .gate_id = GATE_IWDG2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_iwdg2", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_iwdg3 = { + .gate_id = GATE_IWDG3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_iwdg3", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_iwdg4 = { + .gate_id = GATE_IWDG4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_iwdg4", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_iwdg5 = { + .gate_id = GATE_IWDG5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_iwdg5", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* LPTIM */ +static struct clk_stm32_gate ck_icn_p_lptim1 = { + .gate_id = GATE_LPTIM1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lptim1", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_lptim2 = { + .gate_id = GATE_LPTIM2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lptim2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_lptim3 = { + .gate_id = GATE_LPTIM3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lptim3", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_lptim4 = { + .gate_id = GATE_LPTIM4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lptim4", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_lptim5 = { + .gate_id = GATE_LPTIM5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lptim5", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lptim1 = { + .gate_id = GATE_LPTIM1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lptim1", FLEXGEN_07, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lptim2 = { + .gate_id = GATE_LPTIM2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lptim2", FLEXGEN_07, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lptim3 = { + .gate_id = GATE_LPTIM3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lptim3", FLEXGEN_40, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lptim4 = { + .gate_id = GATE_LPTIM4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lptim4", FLEXGEN_41, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lptim5 = { + .gate_id = GATE_LPTIM5, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lptim5", FLEXGEN_41, &clk_stm32_gate_ops, 0), +}; + +/* LPUART */ +static struct clk_stm32_gate ck_icn_p_lpuart1 = { + .gate_id = GATE_LPUART1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lpuart1", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lpuart1 = { + .gate_id = GATE_LPUART1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lpuart1", FLEXGEN_39, &clk_stm32_gate_ops, 0), +}; + +/* MCO1 & MCO2 */ +static struct clk_stm32_composite ck_mco1 = { + .gate_id = GATE_MCO1, + .mux_id = MUX_MCO1, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_mco1", mco1_src, &clk_stm32_composite_ops, 0), +}; + +static struct clk_stm32_composite ck_mco2 = { + .gate_id = GATE_MCO2, + .mux_id = MUX_MCO2, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_mco2", mco2_src, &clk_stm32_composite_ops, 0), +}; + +/* MDF */ +static struct clk_stm32_gate ck_icn_p_mdf1 = { + .gate_id = GATE_MDF1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_mdf1", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_mdf1 = { + .gate_id = GATE_MDF1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_mdf1", FLEXGEN_23, &clk_stm32_gate_ops, 0), +}; + +/* OSPI */ +static struct clk_stm32_gate ck_icn_p_ospiiom = { + .gate_id = GATE_OSPIIOM, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_ospiiom", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* PCIE */ +static struct clk_stm32_gate ck_icn_p_pcie = { + .gate_id = GATE_PCIE, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_pcie", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* SAI */ +static struct clk_stm32_gate ck_icn_p_sai1 = { + .gate_id = GATE_SAI1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_sai1", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_sai2 = { + .gate_id = GATE_SAI2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_sai2", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_sai3 = { + .gate_id = GATE_SAI3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_sai3", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_sai4 = { + .gate_id = GATE_SAI4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_sai4", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_sai1 = { + .gate_id = GATE_SAI1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sai1", FLEXGEN_23, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_sai2 = { + .gate_id = GATE_SAI2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sai2", FLEXGEN_24, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_sai3 = { + .gate_id = GATE_SAI3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sai3", FLEXGEN_25, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_sai4 = { + .gate_id = GATE_SAI4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sai4", FLEXGEN_25, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +/* SDMMC */ +static struct clk_stm32_gate ck_icn_m_sdmmc1 = { + .gate_id = GATE_SDMMC1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_sdmmc1", ICN_SDMMC, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_m_sdmmc2 = { + .gate_id = GATE_SDMMC2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_sdmmc2", ICN_SDMMC, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_m_sdmmc3 = { + .gate_id = GATE_SDMMC3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_sdmmc3", ICN_SDMMC, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_sdmmc1 = { + .gate_id = GATE_SDMMC1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sdmmc1", FLEXGEN_51, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_sdmmc2 = { + .gate_id = GATE_SDMMC2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sdmmc2", FLEXGEN_52, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_sdmmc3 = { + .gate_id = GATE_SDMMC3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sdmmc3", FLEXGEN_53, &clk_stm32_gate_ops, 0), +}; + +/* SPDIF */ +static struct clk_stm32_gate ck_icn_p_spdifrx = { + .gate_id = GATE_SPDIFRX, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spdifrx", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spdifrx = { + .gate_id = GATE_SPDIFRX, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spdifrx", FLEXGEN_11, &clk_stm32_gate_ops, 0), +}; + +/* SPI */ +static struct clk_stm32_gate ck_icn_p_spi1 = { + .gate_id = GATE_SPI1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi1", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi2 = { + .gate_id = GATE_SPI2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi3 = { + .gate_id = GATE_SPI3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi3", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi4 = { + .gate_id = GATE_SPI4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi4", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi5 = { + .gate_id = GATE_SPI5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi5", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi6 = { + .gate_id = GATE_SPI6, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi6", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi7 = { + .gate_id = GATE_SPI7, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi7", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi8 = { + .gate_id = GATE_SPI8, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi8", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spi1 = { + .gate_id = GATE_SPI1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi1", FLEXGEN_16, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_spi2 = { + .gate_id = GATE_SPI2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi2", FLEXGEN_10, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_spi3 = { + .gate_id = GATE_SPI3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi3", FLEXGEN_10, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_spi4 = { + .gate_id = GATE_SPI4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi4", FLEXGEN_17, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spi5 = { + .gate_id = GATE_SPI5, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi5", FLEXGEN_17, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spi6 = { + .gate_id = GATE_SPI6, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi6", FLEXGEN_18, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spi7 = { + .gate_id = GATE_SPI7, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi7", FLEXGEN_18, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spi8 = { + .gate_id = GATE_SPI8, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi8", FLEXGEN_37, &clk_stm32_gate_ops, 0), +}; + +/* Timers */ +static struct clk_stm32_gate ck_icn_p_tim2 = { + .gate_id = GATE_TIM2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim3 = { + .gate_id = GATE_TIM3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim3", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim4 = { + .gate_id = GATE_TIM4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim4", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim5 = { + .gate_id = GATE_TIM5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim5", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim6 = { + .gate_id = GATE_TIM6, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim6", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim7 = { + .gate_id = GATE_TIM7, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim7", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim10 = { + .gate_id = GATE_TIM10, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim10", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim11 = { + .gate_id = GATE_TIM11, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim11", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim12 = { + .gate_id = GATE_TIM12, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim12", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim13 = { + .gate_id = GATE_TIM13, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim13", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim14 = { + .gate_id = GATE_TIM14, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim14", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim1 = { + .gate_id = GATE_TIM1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim1", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim8 = { + .gate_id = GATE_TIM8, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim8", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim15 = { + .gate_id = GATE_TIM15, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim15", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim16 = { + .gate_id = GATE_TIM16, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim16", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim17 = { + .gate_id = GATE_TIM17, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim17", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim20 = { + .gate_id = GATE_TIM20, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim20", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim2 = { + .gate_id = GATE_TIM2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim2", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim3 = { + .gate_id = GATE_TIM3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim3", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim4 = { + .gate_id = GATE_TIM4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim4", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim5 = { + .gate_id = GATE_TIM5, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim5", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim6 = { + .gate_id = GATE_TIM6, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim6", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim7 = { + .gate_id = GATE_TIM7, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim7", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim10 = { + .gate_id = GATE_TIM10, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim10", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim11 = { + .gate_id = GATE_TIM11, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim11", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim12 = { + .gate_id = GATE_TIM12, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim12", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim13 = { + .gate_id = GATE_TIM13, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim13", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim14 = { + .gate_id = GATE_TIM14, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim14", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim1 = { + .gate_id = GATE_TIM1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim1", TIMG2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim8 = { + .gate_id = GATE_TIM8, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim8", TIMG2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim15 = { + .gate_id = GATE_TIM15, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim15", TIMG2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim16 = { + .gate_id = GATE_TIM16, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim16", TIMG2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim17 = { + .gate_id = GATE_TIM17, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim17", TIMG2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim20 = { + .gate_id = GATE_TIM20, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim20", TIMG2, &clk_stm32_gate_ops, 0), +}; + +/* UART/USART */ +static struct clk_stm32_gate ck_icn_p_usart2 = { + .gate_id = GATE_USART2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usart2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_usart3 = { + .gate_id = GATE_USART3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usart3", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_uart4 = { + .gate_id = GATE_UART4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_uart4", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_uart5 = { + .gate_id = GATE_UART5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_uart5", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_usart1 = { + .gate_id = GATE_USART1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usart1", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_usart6 = { + .gate_id = GATE_USART6, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usart6", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_uart7 = { + .gate_id = GATE_UART7, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_uart7", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_uart8 = { + .gate_id = GATE_UART8, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_uart8", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_uart9 = { + .gate_id = GATE_UART9, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_uart9", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usart2 = { + .gate_id = GATE_USART2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usart2", FLEXGEN_08, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_uart4 = { + .gate_id = GATE_UART4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_uart4", FLEXGEN_08, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usart3 = { + .gate_id = GATE_USART3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usart3", FLEXGEN_09, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_uart5 = { + .gate_id = GATE_UART5, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_uart5", FLEXGEN_09, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usart1 = { + .gate_id = GATE_USART1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usart1", FLEXGEN_19, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usart6 = { + .gate_id = GATE_USART6, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usart6", FLEXGEN_20, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_uart7 = { + .gate_id = GATE_UART7, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_uart7", FLEXGEN_21, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_uart8 = { + .gate_id = GATE_UART8, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_uart8", FLEXGEN_21, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_uart9 = { + .gate_id = GATE_UART9, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_uart9", FLEXGEN_22, &clk_stm32_gate_ops, 0), +}; + +/* USB2PHY1 */ +static struct clk_stm32_composite ck_ker_usb2phy1 = { + .gate_id = GATE_USB2PHY1, + .mux_id = MUX_USB2PHY1, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_usb2phy1", usb2phy1_src, + &clk_stm32_composite_ops, 0), +}; + +/* USB2H */ +static struct clk_stm32_gate ck_icn_m_usb2ehci = { + .gate_id = GATE_USBH, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_usb2ehci", ICN_HSL, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_m_usb2ohci = { + .gate_id = GATE_USBH, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_usb2ohci", ICN_HSL, &clk_stm32_gate_ops, 0), +}; + +/* USB2PHY2 */ +static struct clk_stm32_composite ck_ker_usb2phy2_en = { + .gate_id = GATE_USB2PHY2, + .mux_id = MUX_USB2PHY2, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_usb2phy2_en", usb2phy2_src, + &clk_stm32_composite_ops, 0), +}; + +/* USB3 PCIe COMBOPHY */ +static struct clk_stm32_gate ck_icn_p_usb3pciephy = { + .gate_id = GATE_USB3PCIEPHY, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usb3pciephy", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_composite ck_ker_usb3pciephy = { + .gate_id = GATE_USB3PCIEPHY, + .mux_id = MUX_USB3PCIEPHY, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_usb3pciephy", usb3pciphy_src, + &clk_stm32_composite_ops, 0), +}; + +/* USB3 DRD */ +static struct clk_stm32_gate ck_icn_m_usb3dr = { + .gate_id = GATE_USB3DR, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_usb3dr", ICN_HSL, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usb2phy2 = { + .gate_id = GATE_USB3DR, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usb2phy2", FLEXGEN_58, &clk_stm32_gate_ops, 0), +}; + +/* USBTC */ +static struct clk_stm32_gate ck_icn_p_usbtc = { + .gate_id = GATE_USBTC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usbtc", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usbtc = { + .gate_id = GATE_USBTC, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usbtc", FLEXGEN_35, &clk_stm32_gate_ops, 0), +}; + +/* VDEC / VENC */ +static struct clk_stm32_gate ck_icn_p_vdec = { + .gate_id = GATE_VDEC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_vdec", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_venc = { + .gate_id = GATE_VENC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_venc", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +/* VREF */ +static struct clk_stm32_gate ck_icn_p_vref = { + .gate_id = GATE_VREF, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_vref", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +/* WWDG */ +static struct clk_stm32_gate ck_icn_p_wwdg1 = { + .gate_id = GATE_WWDG1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_wwdg1", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_wwdg2 = { + .gate_id = GATE_WWDG2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_wwdg2", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +#define SECF_NONE -1 + +static const struct clock_config stm32mp25_clock_cfg[] = { + STM32_GATE_CFG(CK_BUS_ETH1, ck_icn_p_eth1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ETH2, ck_icn_p_eth2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_PCIE, ck_icn_p_pcie, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ETHSW, ck_icn_p_ethsw, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ADC12, ck_icn_p_adc12, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ADC3, ck_icn_p_adc3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_CCI, ck_icn_p_cci, SECF_NONE), + STM32_GATE_CFG(CK_BUS_CRC, ck_icn_p_crc, SECF_NONE), + STM32_GATE_CFG(CK_BUS_MDF1, ck_icn_p_mdf1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_OSPIIOM, ck_icn_p_ospiiom, SECF_NONE), + STM32_GATE_CFG(CK_BUS_HASH, ck_icn_p_hash, SECF_NONE), + STM32_GATE_CFG(CK_BUS_CRYP1, ck_icn_p_cryp1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_CRYP2, ck_icn_p_cryp2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ADF1, ck_icn_p_adf1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI8, ck_icn_p_spi8, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPUART1, ck_icn_p_lpuart1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C8, ck_icn_p_i2c8, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPTIM3, ck_icn_p_lptim3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPTIM4, ck_icn_p_lptim4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPTIM5, ck_icn_p_lptim5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_IWDG5, ck_icn_p_iwdg5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_WWDG2, ck_icn_p_wwdg2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I3C4, ck_icn_p_i3c4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SDMMC1, ck_icn_m_sdmmc1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SDMMC2, ck_icn_m_sdmmc2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SDMMC3, ck_icn_m_sdmmc3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USB2OHCI, ck_icn_m_usb2ohci, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USB2EHCI, ck_icn_m_usb2ehci, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USB3DR, ck_icn_m_usb3dr, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM2, ck_icn_p_tim2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM3, ck_icn_p_tim3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM4, ck_icn_p_tim4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM5, ck_icn_p_tim5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM6, ck_icn_p_tim6, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM7, ck_icn_p_tim7, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM10, ck_icn_p_tim10, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM11, ck_icn_p_tim11, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM12, ck_icn_p_tim12, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM13, ck_icn_p_tim13, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM14, ck_icn_p_tim14, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPTIM1, ck_icn_p_lptim1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPTIM2, ck_icn_p_lptim2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI2, ck_icn_p_spi2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI3, ck_icn_p_spi3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPDIFRX, ck_icn_p_spdifrx, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USART2, ck_icn_p_usart2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USART3, ck_icn_p_usart3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_UART4, ck_icn_p_uart4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_UART5, ck_icn_p_uart5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C1, ck_icn_p_i2c1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C2, ck_icn_p_i2c2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C3, ck_icn_p_i2c3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C4, ck_icn_p_i2c4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C5, ck_icn_p_i2c5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C6, ck_icn_p_i2c6, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C7, ck_icn_p_i2c7, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I3C1, ck_icn_p_i3c1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I3C2, ck_icn_p_i3c2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I3C3, ck_icn_p_i3c3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM1, ck_icn_p_tim1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM8, ck_icn_p_tim8, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM15, ck_icn_p_tim15, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM16, ck_icn_p_tim16, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM17, ck_icn_p_tim17, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM20, ck_icn_p_tim20, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SAI1, ck_icn_p_sai1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SAI2, ck_icn_p_sai2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SAI3, ck_icn_p_sai3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SAI4, ck_icn_p_sai4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USART1, ck_icn_p_usart1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USART6, ck_icn_p_usart6, SECF_NONE), + STM32_GATE_CFG(CK_BUS_UART7, ck_icn_p_uart7, SECF_NONE), + STM32_GATE_CFG(CK_BUS_UART8, ck_icn_p_uart8, SECF_NONE), + STM32_GATE_CFG(CK_BUS_UART9, ck_icn_p_uart9, SECF_NONE), + STM32_GATE_CFG(CK_BUS_FDCAN, ck_icn_p_fdcan, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI1, ck_icn_p_spi1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI4, ck_icn_p_spi4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI5, ck_icn_p_spi5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI6, ck_icn_p_spi6, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI7, ck_icn_p_spi7, SECF_NONE), + STM32_GATE_CFG(CK_BUS_IWDG2, ck_icn_p_iwdg2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_IWDG3, ck_icn_p_iwdg3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_IWDG4, ck_icn_p_iwdg4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_WWDG1, ck_icn_p_wwdg1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_VREF, ck_icn_p_vref, SECF_NONE), + STM32_GATE_CFG(CK_BUS_HDP, ck_icn_p_hdp, SECF_NONE), + STM32_GATE_CFG(CK_BUS_IS2M, ck_icn_p_is2m, SECF_NONE), + STM32_GATE_CFG(CK_BUS_DSI, ck_icn_p_dsi, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LTDC, ck_icn_p_ltdc, SECF_NONE), + STM32_GATE_CFG(CK_BUS_CSI, ck_icn_p_csi, SECF_NONE), + STM32_GATE_CFG(CK_BUS_DCMIPP, ck_icn_p_dcmipp, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LVDS, ck_icn_p_lvds, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USBTC, ck_icn_p_usbtc, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USB3PCIEPHY, ck_icn_p_usb3pciephy, SECF_NONE), + STM32_GATE_CFG(CK_BUS_VDEC, ck_icn_p_vdec, SECF_NONE), + STM32_GATE_CFG(CK_BUS_VENC, ck_icn_p_venc, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM2, ck_ker_tim2, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM3, ck_ker_tim3, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM4, ck_ker_tim4, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM5, ck_ker_tim5, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM6, ck_ker_tim6, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM7, ck_ker_tim7, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM10, ck_ker_tim10, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM11, ck_ker_tim11, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM12, ck_ker_tim12, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM13, ck_ker_tim13, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM14, ck_ker_tim14, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM1, ck_ker_tim1, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM8, ck_ker_tim8, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM15, ck_ker_tim15, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM16, ck_ker_tim16, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM17, ck_ker_tim17, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM20, ck_ker_tim20, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPTIM1, ck_ker_lptim1, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPTIM2, ck_ker_lptim2, SECF_NONE), + STM32_GATE_CFG(CK_KER_USART2, ck_ker_usart2, SECF_NONE), + STM32_GATE_CFG(CK_KER_UART4, ck_ker_uart4, SECF_NONE), + STM32_GATE_CFG(CK_KER_USART3, ck_ker_usart3, SECF_NONE), + STM32_GATE_CFG(CK_KER_UART5, ck_ker_uart5, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI2, ck_ker_spi2, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI3, ck_ker_spi3, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPDIFRX, ck_ker_spdifrx, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C1, ck_ker_i2c1, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C2, ck_ker_i2c2, SECF_NONE), + STM32_GATE_CFG(CK_KER_I3C1, ck_ker_i3c1, SECF_NONE), + STM32_GATE_CFG(CK_KER_I3C2, ck_ker_i3c2, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C3, ck_ker_i2c3, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C5, ck_ker_i2c5, SECF_NONE), + STM32_GATE_CFG(CK_KER_I3C3, ck_ker_i3c3, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C4, ck_ker_i2c4, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C6, ck_ker_i2c6, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C7, ck_ker_i2c7, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI1, ck_ker_spi1, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI4, ck_ker_spi4, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI5, ck_ker_spi5, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI6, ck_ker_spi6, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI7, ck_ker_spi7, SECF_NONE), + STM32_GATE_CFG(CK_KER_USART1, ck_ker_usart1, SECF_NONE), + STM32_GATE_CFG(CK_KER_USART6, ck_ker_usart6, SECF_NONE), + STM32_GATE_CFG(CK_KER_UART7, ck_ker_uart7, SECF_NONE), + STM32_GATE_CFG(CK_KER_UART8, ck_ker_uart8, SECF_NONE), + STM32_GATE_CFG(CK_KER_UART9, ck_ker_uart9, SECF_NONE), + STM32_GATE_CFG(CK_KER_MDF1, ck_ker_mdf1, SECF_NONE), + STM32_GATE_CFG(CK_KER_SAI1, ck_ker_sai1, SECF_NONE), + STM32_GATE_CFG(CK_KER_SAI2, ck_ker_sai2, SECF_NONE), + STM32_GATE_CFG(CK_KER_SAI3, ck_ker_sai3, SECF_NONE), + STM32_GATE_CFG(CK_KER_SAI4, ck_ker_sai4, SECF_NONE), + STM32_GATE_CFG(CK_KER_FDCAN, ck_ker_fdcan, SECF_NONE), + STM32_GATE_CFG(CK_KER_CSI, ck_ker_csi, SECF_NONE), + STM32_GATE_CFG(CK_KER_CSITXESC, ck_ker_csitxesc, SECF_NONE), + STM32_GATE_CFG(CK_KER_CSIPHY, ck_ker_csiphy, SECF_NONE), + STM32_GATE_CFG(CK_KER_USBTC, ck_ker_usbtc, SECF_NONE), + STM32_GATE_CFG(CK_KER_I3C4, ck_ker_i3c4, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI8, ck_ker_spi8, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C8, ck_ker_i2c8, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPUART1, ck_ker_lpuart1, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPTIM3, ck_ker_lptim3, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPTIM4, ck_ker_lptim4, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPTIM5, ck_ker_lptim5, SECF_NONE), + STM32_GATE_CFG(CK_KER_ADF1, ck_ker_adf1, SECF_NONE), + STM32_GATE_CFG(CK_KER_SDMMC1, ck_ker_sdmmc1, SECF_NONE), + STM32_GATE_CFG(CK_KER_SDMMC2, ck_ker_sdmmc2, SECF_NONE), + STM32_GATE_CFG(CK_KER_SDMMC3, ck_ker_sdmmc3, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETH1, ck_ker_eth1, SECF_NONE), + STM32_GATE_CFG(CK_ETH1_STP, ck_ker_eth1stp, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETHSW, ck_ker_ethsw, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETH2, ck_ker_eth2, SECF_NONE), + STM32_GATE_CFG(CK_ETH2_STP, ck_ker_eth2stp, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETH1PTP, ck_ker_eth1ptp, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETH2PTP, ck_ker_eth2ptp, SECF_NONE), + STM32_GATE_CFG(CK_BUS_GPU, ck_icn_m_gpu, SECF_NONE), + STM32_GATE_CFG(CK_KER_GPU, ck_ker_gpu, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETHSWREF, ck_ker_ethswref, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ETHSWACMCFG, ck_icn_p_ethsw_acm_cfg, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ETHSWACMMSG, ck_icn_p_ethsw_acm_msg, SECF_NONE), + STM32_GATE_CFG(CK_ETH1_MAC, ck_ker_eth1mac, SECF_NONE), + STM32_GATE_CFG(CK_ETH1_TX, ck_ker_eth1tx, SECF_NONE), + STM32_GATE_CFG(CK_ETH1_RX, ck_ker_eth1rx, SECF_NONE), + STM32_GATE_CFG(CK_ETH2_MAC, ck_ker_eth2mac, SECF_NONE), + STM32_GATE_CFG(CK_ETH2_TX, ck_ker_eth2tx, SECF_NONE), + STM32_GATE_CFG(CK_ETH2_RX, ck_ker_eth2rx, SECF_NONE), + STM32_COMPOSITE_CFG(CK_MCO1, ck_mco1, SECF_NONE), + STM32_COMPOSITE_CFG(CK_MCO2, ck_mco2, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_ADC12, ck_ker_adc12, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_ADC3, ck_ker_adc3, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_USB2PHY1, ck_ker_usb2phy1, SECF_NONE), + STM32_GATE_CFG(CK_KER_USB2PHY2, ck_ker_usb2phy2, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_USB2PHY2EN, ck_ker_usb2phy2_en, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_USB3PCIEPHY, ck_ker_usb3pciephy, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_DSIBLANE, clk_lanebyte, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_DSIPHY, clk_phy_dsi, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_LVDSPHY, ck_ker_lvdsphy, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_DTS, ck_ker_dts, SECF_NONE), + STM32_GATE_CFG(CK_KER_LTDC, ck_ker_ltdc, SECF_NONE), +}; + +#define RESET_MP25(id, _offset, _bit_idx, _set_clr) \ + [id] = &(struct stm32_reset_cfg){ \ + .offset = (_offset), \ + .bit_idx = (_bit_idx), \ + .set_clr = (_set_clr), \ + } + +static const struct stm32_reset_cfg *stm32mp25_reset_cfg[STM32MP25_LAST_RESET] = { + RESET_MP25(TIM1_R, RCC_TIM1CFGR, 0, 0), + RESET_MP25(TIM2_R, RCC_TIM2CFGR, 0, 0), + RESET_MP25(TIM3_R, RCC_TIM3CFGR, 0, 0), + RESET_MP25(TIM4_R, RCC_TIM4CFGR, 0, 0), + RESET_MP25(TIM5_R, RCC_TIM5CFGR, 0, 0), + RESET_MP25(TIM6_R, RCC_TIM6CFGR, 0, 0), + RESET_MP25(TIM7_R, RCC_TIM7CFGR, 0, 0), + RESET_MP25(TIM8_R, RCC_TIM8CFGR, 0, 0), + RESET_MP25(TIM10_R, RCC_TIM10CFGR, 0, 0), + RESET_MP25(TIM11_R, RCC_TIM11CFGR, 0, 0), + RESET_MP25(TIM12_R, RCC_TIM12CFGR, 0, 0), + RESET_MP25(TIM13_R, RCC_TIM13CFGR, 0, 0), + RESET_MP25(TIM14_R, RCC_TIM14CFGR, 0, 0), + RESET_MP25(TIM15_R, RCC_TIM15CFGR, 0, 0), + RESET_MP25(TIM16_R, RCC_TIM16CFGR, 0, 0), + RESET_MP25(TIM17_R, RCC_TIM17CFGR, 0, 0), + RESET_MP25(TIM20_R, RCC_TIM20CFGR, 0, 0), + RESET_MP25(LPTIM1_R, RCC_LPTIM1CFGR, 0, 0), + RESET_MP25(LPTIM2_R, RCC_LPTIM2CFGR, 0, 0), + RESET_MP25(LPTIM3_R, RCC_LPTIM3CFGR, 0, 0), + RESET_MP25(LPTIM4_R, RCC_LPTIM4CFGR, 0, 0), + RESET_MP25(LPTIM5_R, RCC_LPTIM5CFGR, 0, 0), + RESET_MP25(SPI1_R, RCC_SPI1CFGR, 0, 0), + RESET_MP25(SPI2_R, RCC_SPI2CFGR, 0, 0), + RESET_MP25(SPI3_R, RCC_SPI3CFGR, 0, 0), + RESET_MP25(SPI4_R, RCC_SPI4CFGR, 0, 0), + RESET_MP25(SPI5_R, RCC_SPI5CFGR, 0, 0), + RESET_MP25(SPI6_R, RCC_SPI6CFGR, 0, 0), + RESET_MP25(SPI7_R, RCC_SPI7CFGR, 0, 0), + RESET_MP25(SPI8_R, RCC_SPI8CFGR, 0, 0), + RESET_MP25(SPDIFRX_R, RCC_SPDIFRXCFGR, 0, 0), + RESET_MP25(USART1_R, RCC_USART1CFGR, 0, 0), + RESET_MP25(USART2_R, RCC_USART2CFGR, 0, 0), + RESET_MP25(USART3_R, RCC_USART3CFGR, 0, 0), + RESET_MP25(UART4_R, RCC_UART4CFGR, 0, 0), + RESET_MP25(UART5_R, RCC_UART5CFGR, 0, 0), + RESET_MP25(USART6_R, RCC_USART6CFGR, 0, 0), + RESET_MP25(UART7_R, RCC_UART7CFGR, 0, 0), + RESET_MP25(UART8_R, RCC_UART8CFGR, 0, 0), + RESET_MP25(UART9_R, RCC_UART9CFGR, 0, 0), + RESET_MP25(LPUART1_R, RCC_LPUART1CFGR, 0, 0), + RESET_MP25(IS2M_R, RCC_IS2MCFGR, 0, 0), + RESET_MP25(I2C1_R, RCC_I2C1CFGR, 0, 0), + RESET_MP25(I2C2_R, RCC_I2C2CFGR, 0, 0), + RESET_MP25(I2C3_R, RCC_I2C3CFGR, 0, 0), + RESET_MP25(I2C4_R, RCC_I2C4CFGR, 0, 0), + RESET_MP25(I2C5_R, RCC_I2C5CFGR, 0, 0), + RESET_MP25(I2C6_R, RCC_I2C6CFGR, 0, 0), + RESET_MP25(I2C7_R, RCC_I2C7CFGR, 0, 0), + RESET_MP25(I2C8_R, RCC_I2C8CFGR, 0, 0), + RESET_MP25(SAI1_R, RCC_SAI1CFGR, 0, 0), + RESET_MP25(SAI2_R, RCC_SAI2CFGR, 0, 0), + RESET_MP25(SAI3_R, RCC_SAI3CFGR, 0, 0), + RESET_MP25(SAI4_R, RCC_SAI4CFGR, 0, 0), + RESET_MP25(MDF1_R, RCC_MDF1CFGR, 0, 0), + RESET_MP25(MDF2_R, RCC_ADF1CFGR, 0, 0), + RESET_MP25(FDCAN_R, RCC_FDCANCFGR, 0, 0), + RESET_MP25(HDP_R, RCC_HDPCFGR, 0, 0), + RESET_MP25(ADC12_R, RCC_ADC12CFGR, 0, 0), + RESET_MP25(ADC3_R, RCC_ADC3CFGR, 0, 0), + RESET_MP25(ETH1_R, RCC_ETH1CFGR, 0, 0), + RESET_MP25(ETH2_R, RCC_ETH2CFGR, 0, 0), + RESET_MP25(USBH_R, RCC_USBHCFGR, 0, 0), + RESET_MP25(USB2PHY1_R, RCC_USB2PHY1CFGR, 0, 0), + RESET_MP25(USB2PHY2_R, RCC_USB2PHY2CFGR, 0, 0), + RESET_MP25(USB3DR_R, RCC_USB3DRCFGR, 0, 0), + RESET_MP25(USB3PCIEPHY_R, RCC_USB3PCIEPHYCFGR, 0, 0), + RESET_MP25(USBTC_R, RCC_USBTCCFGR, 0, 0), + RESET_MP25(ETHSW_R, RCC_ETHSWCFGR, 0, 0), + RESET_MP25(SDMMC1_R, RCC_SDMMC1CFGR, 0, 0), + RESET_MP25(SDMMC1DLL_R, RCC_SDMMC1CFGR, 16, 0), + RESET_MP25(SDMMC2_R, RCC_SDMMC2CFGR, 0, 0), + RESET_MP25(SDMMC2DLL_R, RCC_SDMMC2CFGR, 16, 0), + RESET_MP25(SDMMC3_R, RCC_SDMMC3CFGR, 0, 0), + RESET_MP25(SDMMC3DLL_R, RCC_SDMMC3CFGR, 16, 0), + RESET_MP25(GPU_R, RCC_GPUCFGR, 0, 0), + RESET_MP25(LTDC_R, RCC_LTDCCFGR, 0, 0), + RESET_MP25(DSI_R, RCC_DSICFGR, 0, 0), + RESET_MP25(LVDS_R, RCC_LVDSCFGR, 0, 0), + RESET_MP25(CSI_R, RCC_CSICFGR, 0, 0), + RESET_MP25(DCMIPP_R, RCC_DCMIPPCFGR, 0, 0), + RESET_MP25(CCI_R, RCC_CCICFGR, 0, 0), + RESET_MP25(VDEC_R, RCC_VDECCFGR, 0, 0), + RESET_MP25(VENC_R, RCC_VENCCFGR, 0, 0), + RESET_MP25(WWDG1_R, RCC_WWDG1CFGR, 0, 0), + RESET_MP25(WWDG2_R, RCC_WWDG2CFGR, 0, 0), + RESET_MP25(VREF_R, RCC_VREFCFGR, 0, 0), + RESET_MP25(DTS_R, RCC_DTSCFGR, 0, 0), + RESET_MP25(CRC_R, RCC_CRCCFGR, 0, 0), + RESET_MP25(SERC_R, RCC_SERCCFGR, 0, 0), + RESET_MP25(OSPIIOM_R, RCC_OSPIIOMCFGR, 0, 0), + RESET_MP25(I3C1_R, RCC_I3C1CFGR, 0, 0), + RESET_MP25(I3C2_R, RCC_I3C2CFGR, 0, 0), + RESET_MP25(I3C3_R, RCC_I3C3CFGR, 0, 0), + RESET_MP25(I3C4_R, RCC_I3C4CFGR, 0, 0), + RESET_MP25(IWDG2_KER_R, RCC_IWDGC1CFGSETR, 18, 1), + RESET_MP25(IWDG4_KER_R, RCC_IWDGC2CFGSETR, 18, 1), + RESET_MP25(RNG_R, RCC_RNGCFGR, 0, 0), + RESET_MP25(PKA_R, RCC_PKACFGR, 0, 0), + RESET_MP25(SAES_R, RCC_SAESCFGR, 0, 0), + RESET_MP25(HASH_R, RCC_HASHCFGR, 0, 0), + RESET_MP25(CRYP1_R, RCC_CRYP1CFGR, 0, 0), + RESET_MP25(CRYP2_R, RCC_CRYP2CFGR, 0, 0), + RESET_MP25(PCIE_R, RCC_PCIECFGR, 0, 0), +}; + +static u16 stm32mp25_cpt_gate[GATE_NB]; + +static struct clk_stm32_clock_data stm32mp25_clock_data = { + .gate_cpt = stm32mp25_cpt_gate, + .gates = stm32mp25_gates, + .muxes = stm32mp25_muxes, +}; + +static struct clk_stm32_reset_data stm32mp25_reset_data = { + .reset_lines = stm32mp25_reset_cfg, + .nr_lines = ARRAY_SIZE(stm32mp25_reset_cfg), +}; + +static const struct stm32_rcc_match_data stm32mp25_data = { + .tab_clocks = stm32mp25_clock_cfg, + .num_clocks = ARRAY_SIZE(stm32mp25_clock_cfg), + .maxbinding = STM32MP25_LAST_CLK, + .clock_data = &stm32mp25_clock_data, + .reset_data = &stm32mp25_reset_data, +}; + +static const struct of_device_id stm32mp25_match_data[] = { + { .compatible = "st,stm32mp25-rcc", .data = &stm32mp25_data, }, + { } +}; +MODULE_DEVICE_TABLE(of, stm32mp25_match_data); + +static int stm32mp25_rcc_clocks_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + void __iomem *base; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + return stm32_rcc_init(dev, stm32mp25_match_data, base); +} + +static struct platform_driver stm32mp25_rcc_clocks_driver = { + .driver = { + .name = "stm32mp25_rcc", + .of_match_table = stm32mp25_match_data, + }, + .probe = stm32mp25_rcc_clocks_probe, +}; + +static int __init stm32mp25_clocks_init(void) +{ + return platform_driver_register(&stm32mp25_rcc_clocks_driver); +} + +core_initcall(stm32mp25_clocks_init); diff --git a/drivers/clk/stm32/reset-stm32.c b/drivers/clk/stm32/reset-stm32.c index 14c2ee1eebee..5a8f525842ce 100644 --- a/drivers/clk/stm32/reset-stm32.c +++ b/drivers/clk/stm32/reset-stm32.c @@ -19,6 +19,7 @@ struct stm32_reset_data { struct reset_controller_dev rcdev; void __iomem *membase; u32 clear_offset; + const struct stm32_reset_cfg **reset_lines; }; static inline struct stm32_reset_data * @@ -27,22 +28,46 @@ to_stm32_reset_data(struct reset_controller_dev *rcdev) return container_of(rcdev, struct stm32_reset_data, rcdev); } +static const struct stm32_reset_cfg *stm32_get_reset_line(struct reset_controller_dev *rcdev, + unsigned long id, + struct stm32_reset_cfg *line) +{ + struct stm32_reset_data *data = to_stm32_reset_data(rcdev); + + if (!data->reset_lines) { + int reg_width = sizeof(u32); + int bank = id / (reg_width * BITS_PER_BYTE); + int offset = id % (reg_width * BITS_PER_BYTE); + + line->offset = bank * reg_width; + line->bit_idx = offset; + line->set_clr = (data->clear_offset ? true : false); + + return line; + } + + return data->reset_lines[id]; +} + static int stm32_reset_update(struct reset_controller_dev *rcdev, unsigned long id, bool assert) { struct stm32_reset_data *data = to_stm32_reset_data(rcdev); - int reg_width = sizeof(u32); - int bank = id / (reg_width * BITS_PER_BYTE); - int offset = id % (reg_width * BITS_PER_BYTE); + struct stm32_reset_cfg line_reset; + const struct stm32_reset_cfg *ptr_line; - if (data->clear_offset) { + ptr_line = stm32_get_reset_line(rcdev, id, &line_reset); + if (!ptr_line) + return -EPERM; + + if (ptr_line->set_clr) { void __iomem *addr; - addr = data->membase + (bank * reg_width); + addr = data->membase + ptr_line->offset; if (!assert) addr += data->clear_offset; - writel(BIT(offset), addr); + writel(BIT(ptr_line->bit_idx), addr); } else { unsigned long flags; @@ -50,14 +75,14 @@ static int stm32_reset_update(struct reset_controller_dev *rcdev, spin_lock_irqsave(&data->lock, flags); - reg = readl(data->membase + (bank * reg_width)); + reg = readl(data->membase + ptr_line->offset); if (assert) - reg |= BIT(offset); + reg |= BIT(ptr_line->bit_idx); else - reg &= ~BIT(offset); + reg &= ~BIT(ptr_line->bit_idx); - writel(reg, data->membase + (bank * reg_width)); + writel(reg, data->membase + ptr_line->offset); spin_unlock_irqrestore(&data->lock, flags); } @@ -81,14 +106,17 @@ static int stm32_reset_status(struct reset_controller_dev *rcdev, unsigned long id) { struct stm32_reset_data *data = to_stm32_reset_data(rcdev); - int reg_width = sizeof(u32); - int bank = id / (reg_width * BITS_PER_BYTE); - int offset = id % (reg_width * BITS_PER_BYTE); + struct stm32_reset_cfg line_reset; + const struct stm32_reset_cfg *ptr_line; u32 reg; - reg = readl(data->membase + (bank * reg_width)); + ptr_line = stm32_get_reset_line(rcdev, id, &line_reset); + if (!ptr_line) + return -EPERM; + + reg = readl(data->membase + ptr_line->offset); - return !!(reg & BIT(offset)); + return !!(reg & BIT(ptr_line->bit_idx)); } static const struct reset_control_ops stm32_reset_ops = { @@ -113,6 +141,7 @@ int stm32_rcc_reset_init(struct device *dev, struct clk_stm32_reset_data *data, reset_data->rcdev.ops = &stm32_reset_ops; reset_data->rcdev.of_node = dev_of_node(dev); reset_data->rcdev.nr_resets = data->nr_lines; + reset_data->reset_lines = data->reset_lines; reset_data->clear_offset = data->clear_offset; return reset_controller_register(&reset_data->rcdev); diff --git a/drivers/clk/stm32/reset-stm32.h b/drivers/clk/stm32/reset-stm32.h index 8cf1cc9be480..f79cad21dfd6 100644 --- a/drivers/clk/stm32/reset-stm32.h +++ b/drivers/clk/stm32/reset-stm32.h @@ -4,8 +4,15 @@ * Author: Gabriel Fernandez <gabriel.fernandez@foss.st.com> for STMicroelectronics. */ +struct stm32_reset_cfg { + u16 offset; + u8 bit_idx; + bool set_clr; +}; + struct clk_stm32_reset_data { const struct reset_control_ops *ops; + const struct stm32_reset_cfg **reset_lines; unsigned int nr_lines; u32 clear_offset; }; diff --git a/drivers/clk/stm32/stm32mp25_rcc.h b/drivers/clk/stm32/stm32mp25_rcc.h new file mode 100644 index 000000000000..687bc6a78627 --- /dev/null +++ b/drivers/clk/stm32/stm32mp25_rcc.h @@ -0,0 +1,712 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) STMicroelectronics 2023 - All Rights Reserved + * Author: Gabriel Fernandez <gabriel.fernandez@foss.st.com> for STMicroelectronics. + */ + +#ifndef STM32MP25_RCC_H +#define STM32MP25_RCC_H + +#define RCC_SECCFGR0 0x0 +#define RCC_SECCFGR1 0x4 +#define RCC_SECCFGR2 0x8 +#define RCC_SECCFGR3 0xC +#define RCC_PRIVCFGR0 0x10 +#define RCC_PRIVCFGR1 0x14 +#define RCC_PRIVCFGR2 0x18 +#define RCC_PRIVCFGR3 0x1C +#define RCC_RCFGLOCKR0 0x20 +#define RCC_RCFGLOCKR1 0x24 +#define RCC_RCFGLOCKR2 0x28 +#define RCC_RCFGLOCKR3 0x2C +#define RCC_R0CIDCFGR 0x30 +#define RCC_R0SEMCR 0x34 +#define RCC_R1CIDCFGR 0x38 +#define RCC_R1SEMCR 0x3C +#define RCC_R2CIDCFGR 0x40 +#define RCC_R2SEMCR 0x44 +#define RCC_R3CIDCFGR 0x48 +#define RCC_R3SEMCR 0x4C +#define RCC_R4CIDCFGR 0x50 +#define RCC_R4SEMCR 0x54 +#define RCC_R5CIDCFGR 0x58 +#define RCC_R5SEMCR 0x5C +#define RCC_R6CIDCFGR 0x60 +#define RCC_R6SEMCR 0x64 +#define RCC_R7CIDCFGR 0x68 +#define RCC_R7SEMCR 0x6C +#define RCC_R8CIDCFGR 0x70 +#define RCC_R8SEMCR 0x74 +#define RCC_R9CIDCFGR 0x78 +#define RCC_R9SEMCR 0x7C +#define RCC_R10CIDCFGR 0x80 +#define RCC_R10SEMCR 0x84 +#define RCC_R11CIDCFGR 0x88 +#define RCC_R11SEMCR 0x8C +#define RCC_R12CIDCFGR 0x90 +#define RCC_R12SEMCR 0x94 +#define RCC_R13CIDCFGR 0x98 +#define RCC_R13SEMCR 0x9C +#define RCC_R14CIDCFGR 0xA0 +#define RCC_R14SEMCR 0xA4 +#define RCC_R15CIDCFGR 0xA8 +#define RCC_R15SEMCR 0xAC +#define RCC_R16CIDCFGR 0xB0 +#define RCC_R16SEMCR 0xB4 +#define RCC_R17CIDCFGR 0xB8 +#define RCC_R17SEMCR 0xBC +#define RCC_R18CIDCFGR 0xC0 +#define RCC_R18SEMCR 0xC4 +#define RCC_R19CIDCFGR 0xC8 +#define RCC_R19SEMCR 0xCC +#define RCC_R20CIDCFGR 0xD0 +#define RCC_R20SEMCR 0xD4 +#define RCC_R21CIDCFGR 0xD8 +#define RCC_R21SEMCR 0xDC +#define RCC_R22CIDCFGR 0xE0 +#define RCC_R22SEMCR 0xE4 +#define RCC_R23CIDCFGR 0xE8 +#define RCC_R23SEMCR 0xEC +#define RCC_R24CIDCFGR 0xF0 +#define RCC_R24SEMCR 0xF4 +#define RCC_R25CIDCFGR 0xF8 +#define RCC_R25SEMCR 0xFC +#define RCC_R26CIDCFGR 0x100 +#define RCC_R26SEMCR 0x104 +#define RCC_R27CIDCFGR 0x108 +#define RCC_R27SEMCR 0x10C +#define RCC_R28CIDCFGR 0x110 +#define RCC_R28SEMCR 0x114 +#define RCC_R29CIDCFGR 0x118 +#define RCC_R29SEMCR 0x11C +#define RCC_R30CIDCFGR 0x120 +#define RCC_R30SEMCR 0x124 +#define RCC_R31CIDCFGR 0x128 +#define RCC_R31SEMCR 0x12C +#define RCC_R32CIDCFGR 0x130 +#define RCC_R32SEMCR 0x134 +#define RCC_R33CIDCFGR 0x138 +#define RCC_R33SEMCR 0x13C +#define RCC_R34CIDCFGR 0x140 +#define RCC_R34SEMCR 0x144 +#define RCC_R35CIDCFGR 0x148 +#define RCC_R35SEMCR 0x14C +#define RCC_R36CIDCFGR 0x150 +#define RCC_R36SEMCR 0x154 +#define RCC_R37CIDCFGR 0x158 +#define RCC_R37SEMCR 0x15C +#define RCC_R38CIDCFGR 0x160 +#define RCC_R38SEMCR 0x164 +#define RCC_R39CIDCFGR 0x168 +#define RCC_R39SEMCR 0x16C +#define RCC_R40CIDCFGR 0x170 +#define RCC_R40SEMCR 0x174 +#define RCC_R41CIDCFGR 0x178 +#define RCC_R41SEMCR 0x17C +#define RCC_R42CIDCFGR 0x180 +#define RCC_R42SEMCR 0x184 +#define RCC_R43CIDCFGR 0x188 +#define RCC_R43SEMCR 0x18C +#define RCC_R44CIDCFGR 0x190 +#define RCC_R44SEMCR 0x194 +#define RCC_R45CIDCFGR 0x198 +#define RCC_R45SEMCR 0x19C +#define RCC_R46CIDCFGR 0x1A0 +#define RCC_R46SEMCR 0x1A4 +#define RCC_R47CIDCFGR 0x1A8 +#define RCC_R47SEMCR 0x1AC +#define RCC_R48CIDCFGR 0x1B0 +#define RCC_R48SEMCR 0x1B4 +#define RCC_R49CIDCFGR 0x1B8 +#define RCC_R49SEMCR 0x1BC +#define RCC_R50CIDCFGR 0x1C0 +#define RCC_R50SEMCR 0x1C4 +#define RCC_R51CIDCFGR 0x1C8 +#define RCC_R51SEMCR 0x1CC +#define RCC_R52CIDCFGR 0x1D0 +#define RCC_R52SEMCR 0x1D4 +#define RCC_R53CIDCFGR 0x1D8 +#define RCC_R53SEMCR 0x1DC +#define RCC_R54CIDCFGR 0x1E0 +#define RCC_R54SEMCR 0x1E4 +#define RCC_R55CIDCFGR 0x1E8 +#define RCC_R55SEMCR 0x1EC +#define RCC_R56CIDCFGR 0x1F0 +#define RCC_R56SEMCR 0x1F4 +#define RCC_R57CIDCFGR 0x1F8 +#define RCC_R57SEMCR 0x1FC +#define RCC_R58CIDCFGR 0x200 +#define RCC_R58SEMCR 0x204 +#define RCC_R59CIDCFGR 0x208 +#define RCC_R59SEMCR 0x20C +#define RCC_R60CIDCFGR 0x210 +#define RCC_R60SEMCR 0x214 +#define RCC_R61CIDCFGR 0x218 +#define RCC_R61SEMCR 0x21C +#define RCC_R62CIDCFGR 0x220 +#define RCC_R62SEMCR 0x224 +#define RCC_R63CIDCFGR 0x228 +#define RCC_R63SEMCR 0x22C +#define RCC_R64CIDCFGR 0x230 +#define RCC_R64SEMCR 0x234 +#define RCC_R65CIDCFGR 0x238 +#define RCC_R65SEMCR 0x23C +#define RCC_R66CIDCFGR 0x240 +#define RCC_R66SEMCR 0x244 +#define RCC_R67CIDCFGR 0x248 +#define RCC_R67SEMCR 0x24C +#define RCC_R68CIDCFGR 0x250 +#define RCC_R68SEMCR 0x254 +#define RCC_R69CIDCFGR 0x258 +#define RCC_R69SEMCR 0x25C +#define RCC_R70CIDCFGR 0x260 +#define RCC_R70SEMCR 0x264 +#define RCC_R71CIDCFGR 0x268 +#define RCC_R71SEMCR 0x26C +#define RCC_R72CIDCFGR 0x270 +#define RCC_R72SEMCR 0x274 +#define RCC_R73CIDCFGR 0x278 +#define RCC_R73SEMCR 0x27C +#define RCC_R74CIDCFGR 0x280 +#define RCC_R74SEMCR 0x284 +#define RCC_R75CIDCFGR 0x288 +#define RCC_R75SEMCR 0x28C +#define RCC_R76CIDCFGR 0x290 +#define RCC_R76SEMCR 0x294 +#define RCC_R77CIDCFGR 0x298 +#define RCC_R77SEMCR 0x29C +#define RCC_R78CIDCFGR 0x2A0 +#define RCC_R78SEMCR 0x2A4 +#define RCC_R79CIDCFGR 0x2A8 +#define RCC_R79SEMCR 0x2AC +#define RCC_R80CIDCFGR 0x2B0 +#define RCC_R80SEMCR 0x2B4 +#define RCC_R81CIDCFGR 0x2B8 +#define RCC_R81SEMCR 0x2BC +#define RCC_R82CIDCFGR 0x2C0 +#define RCC_R82SEMCR 0x2C4 +#define RCC_R83CIDCFGR 0x2C8 +#define RCC_R83SEMCR 0x2CC +#define RCC_R84CIDCFGR 0x2D0 +#define RCC_R84SEMCR 0x2D4 +#define RCC_R85CIDCFGR 0x2D8 +#define RCC_R85SEMCR 0x2DC +#define RCC_R86CIDCFGR 0x2E0 +#define RCC_R86SEMCR 0x2E4 +#define RCC_R87CIDCFGR 0x2E8 +#define RCC_R87SEMCR 0x2EC +#define RCC_R88CIDCFGR 0x2F0 +#define RCC_R88SEMCR 0x2F4 +#define RCC_R89CIDCFGR 0x2F8 +#define RCC_R89SEMCR 0x2FC +#define RCC_R90CIDCFGR 0x300 +#define RCC_R90SEMCR 0x304 +#define RCC_R91CIDCFGR 0x308 +#define RCC_R91SEMCR 0x30C +#define RCC_R92CIDCFGR 0x310 +#define RCC_R92SEMCR 0x314 +#define RCC_R93CIDCFGR 0x318 +#define RCC_R93SEMCR 0x31C +#define RCC_R94CIDCFGR 0x320 +#define RCC_R94SEMCR 0x324 +#define RCC_R95CIDCFGR 0x328 +#define RCC_R95SEMCR 0x32C +#define RCC_R96CIDCFGR 0x330 +#define RCC_R96SEMCR 0x334 +#define RCC_R97CIDCFGR 0x338 +#define RCC_R97SEMCR 0x33C +#define RCC_R98CIDCFGR 0x340 +#define RCC_R98SEMCR 0x344 +#define RCC_R99CIDCFGR 0x348 +#define RCC_R99SEMCR 0x34C +#define RCC_R100CIDCFGR 0x350 +#define RCC_R100SEMCR 0x354 +#define RCC_R101CIDCFGR 0x358 +#define RCC_R101SEMCR 0x35C +#define RCC_R102CIDCFGR 0x360 +#define RCC_R102SEMCR 0x364 +#define RCC_R103CIDCFGR 0x368 +#define RCC_R103SEMCR 0x36C +#define RCC_R104CIDCFGR 0x370 +#define RCC_R104SEMCR 0x374 +#define RCC_R105CIDCFGR 0x378 +#define RCC_R105SEMCR 0x37C +#define RCC_R106CIDCFGR 0x380 +#define RCC_R106SEMCR 0x384 +#define RCC_R107CIDCFGR 0x388 +#define RCC_R107SEMCR 0x38C +#define RCC_R108CIDCFGR 0x390 +#define RCC_R108SEMCR 0x394 +#define RCC_R109CIDCFGR 0x398 +#define RCC_R109SEMCR 0x39C +#define RCC_R110CIDCFGR 0x3A0 +#define RCC_R110SEMCR 0x3A4 +#define RCC_R111CIDCFGR 0x3A8 +#define RCC_R111SEMCR 0x3AC +#define RCC_R112CIDCFGR 0x3B0 +#define RCC_R112SEMCR 0x3B4 +#define RCC_R113CIDCFGR 0x3B8 +#define RCC_R113SEMCR 0x3BC +#define RCC_GRSTCSETR 0x400 +#define RCC_C1RSTCSETR 0x404 +#define RCC_C1P1RSTCSETR 0x408 +#define RCC_C2RSTCSETR 0x40C +#define RCC_HWRSTSCLRR 0x410 +#define RCC_C1HWRSTSCLRR 0x414 +#define RCC_C2HWRSTSCLRR 0x418 +#define RCC_C1BOOTRSTSSETR 0x41C +#define RCC_C1BOOTRSTSCLRR 0x420 +#define RCC_C2BOOTRSTSSETR 0x424 +#define RCC_C2BOOTRSTSCLRR 0x428 +#define RCC_C1SREQSETR 0x42C +#define RCC_C1SREQCLRR 0x430 +#define RCC_CPUBOOTCR 0x434 +#define RCC_STBYBOOTCR 0x438 +#define RCC_LEGBOOTCR 0x43C +#define RCC_BDCR 0x440 +#define RCC_D3DCR 0x444 +#define RCC_D3DSR 0x448 +#define RCC_RDCR 0x44C +#define RCC_C1MSRDCR 0x450 +#define RCC_PWRLPDLYCR 0x454 +#define RCC_C1CIESETR 0x458 +#define RCC_C1CIFCLRR 0x45C +#define RCC_C2CIESETR 0x460 +#define RCC_C2CIFCLRR 0x464 +#define RCC_IWDGC1FZSETR 0x468 +#define RCC_IWDGC1FZCLRR 0x46C +#define RCC_IWDGC1CFGSETR 0x470 +#define RCC_IWDGC1CFGCLRR 0x474 +#define RCC_IWDGC2FZSETR 0x478 +#define RCC_IWDGC2FZCLRR 0x47C +#define RCC_IWDGC2CFGSETR 0x480 +#define RCC_IWDGC2CFGCLRR 0x484 +#define RCC_IWDGC3CFGSETR 0x488 +#define RCC_IWDGC3CFGCLRR 0x48C +#define RCC_C3CFGR 0x490 +#define RCC_MCO1CFGR 0x494 +#define RCC_MCO2CFGR 0x498 +#define RCC_OCENSETR 0x49C +#define RCC_OCENCLRR 0x4A0 +#define RCC_OCRDYR 0x4A4 +#define RCC_HSICFGR 0x4A8 +#define RCC_MSICFGR 0x4AC +#define RCC_RTCDIVR 0x4B0 +#define RCC_APB1DIVR 0x4B4 +#define RCC_APB2DIVR 0x4B8 +#define RCC_APB3DIVR 0x4BC +#define RCC_APB4DIVR 0x4C0 +#define RCC_APBDBGDIVR 0x4C4 +#define RCC_TIMG1PRER 0x4C8 +#define RCC_TIMG2PRER 0x4CC +#define RCC_LSMCUDIVR 0x4D0 +#define RCC_DDRCPCFGR 0x4D4 +#define RCC_DDRCAPBCFGR 0x4D8 +#define RCC_DDRPHYCAPBCFGR 0x4DC +#define RCC_DDRPHYCCFGR 0x4E0 +#define RCC_DDRCFGR 0x4E4 +#define RCC_DDRITFCFGR 0x4E8 +#define RCC_SYSRAMCFGR 0x4F0 +#define RCC_VDERAMCFGR 0x4F4 +#define RCC_SRAM1CFGR 0x4F8 +#define RCC_SRAM2CFGR 0x4FC +#define RCC_RETRAMCFGR 0x500 +#define RCC_BKPSRAMCFGR 0x504 +#define RCC_LPSRAM1CFGR 0x508 +#define RCC_LPSRAM2CFGR 0x50C +#define RCC_LPSRAM3CFGR 0x510 +#define RCC_OSPI1CFGR 0x514 +#define RCC_OSPI2CFGR 0x518 +#define RCC_FMCCFGR 0x51C +#define RCC_DBGCFGR 0x520 +#define RCC_STM500CFGR 0x524 +#define RCC_ETRCFGR 0x528 +#define RCC_GPIOACFGR 0x52C +#define RCC_GPIOBCFGR 0x530 +#define RCC_GPIOCCFGR 0x534 +#define RCC_GPIODCFGR 0x538 +#define RCC_GPIOECFGR 0x53C +#define RCC_GPIOFCFGR 0x540 +#define RCC_GPIOGCFGR 0x544 +#define RCC_GPIOHCFGR 0x548 +#define RCC_GPIOICFGR 0x54C +#define RCC_GPIOJCFGR 0x550 +#define RCC_GPIOKCFGR 0x554 +#define RCC_GPIOZCFGR 0x558 +#define RCC_HPDMA1CFGR 0x55C +#define RCC_HPDMA2CFGR 0x560 +#define RCC_HPDMA3CFGR 0x564 +#define RCC_LPDMACFGR 0x568 +#define RCC_HSEMCFGR 0x56C +#define RCC_IPCC1CFGR 0x570 +#define RCC_IPCC2CFGR 0x574 +#define RCC_RTCCFGR 0x578 +#define RCC_SYSCPU1CFGR 0x580 +#define RCC_BSECCFGR 0x584 +#define RCC_IS2MCFGR 0x58C +#define RCC_PLL2CFGR1 0x590 +#define RCC_PLL2CFGR2 0x594 +#define RCC_PLL2CFGR3 0x598 +#define RCC_PLL2CFGR4 0x59C +#define RCC_PLL2CFGR5 0x5A0 +#define RCC_PLL2CFGR6 0x5A8 +#define RCC_PLL2CFGR7 0x5AC +#define RCC_PLL3CFGR1 0x5B8 +#define RCC_PLL3CFGR2 0x5BC +#define RCC_PLL3CFGR3 0x5C0 +#define RCC_PLL3CFGR4 0x5C4 +#define RCC_PLL3CFGR5 0x5C8 +#define RCC_PLL3CFGR6 0x5D0 +#define RCC_PLL3CFGR7 0x5D4 +#define RCC_HSIFMONCR 0x5E0 +#define RCC_HSIFVALR 0x5E4 +#define RCC_TIM1CFGR 0x700 +#define RCC_TIM2CFGR 0x704 +#define RCC_TIM3CFGR 0x708 +#define RCC_TIM4CFGR 0x70C +#define RCC_TIM5CFGR 0x710 +#define RCC_TIM6CFGR 0x714 +#define RCC_TIM7CFGR 0x718 +#define RCC_TIM8CFGR 0x71C +#define RCC_TIM10CFGR 0x720 +#define RCC_TIM11CFGR 0x724 +#define RCC_TIM12CFGR 0x728 +#define RCC_TIM13CFGR 0x72C +#define RCC_TIM14CFGR 0x730 +#define RCC_TIM15CFGR 0x734 +#define RCC_TIM16CFGR 0x738 +#define RCC_TIM17CFGR 0x73C +#define RCC_TIM20CFGR 0x740 +#define RCC_LPTIM1CFGR 0x744 +#define RCC_LPTIM2CFGR 0x748 +#define RCC_LPTIM3CFGR 0x74C +#define RCC_LPTIM4CFGR 0x750 +#define RCC_LPTIM5CFGR 0x754 +#define RCC_SPI1CFGR 0x758 +#define RCC_SPI2CFGR 0x75C +#define RCC_SPI3CFGR 0x760 +#define RCC_SPI4CFGR 0x764 +#define RCC_SPI5CFGR 0x768 +#define RCC_SPI6CFGR 0x76C +#define RCC_SPI7CFGR 0x770 +#define RCC_SPI8CFGR 0x774 +#define RCC_SPDIFRXCFGR 0x778 +#define RCC_USART1CFGR 0x77C +#define RCC_USART2CFGR 0x780 +#define RCC_USART3CFGR 0x784 +#define RCC_UART4CFGR 0x788 +#define RCC_UART5CFGR 0x78C +#define RCC_USART6CFGR 0x790 +#define RCC_UART7CFGR 0x794 +#define RCC_UART8CFGR 0x798 +#define RCC_UART9CFGR 0x79C +#define RCC_LPUART1CFGR 0x7A0 +#define RCC_I2C1CFGR 0x7A4 +#define RCC_I2C2CFGR 0x7A8 +#define RCC_I2C3CFGR 0x7AC +#define RCC_I2C4CFGR 0x7B0 +#define RCC_I2C5CFGR 0x7B4 +#define RCC_I2C6CFGR 0x7B8 +#define RCC_I2C7CFGR 0x7BC +#define RCC_I2C8CFGR 0x7C0 +#define RCC_SAI1CFGR 0x7C4 +#define RCC_SAI2CFGR 0x7C8 +#define RCC_SAI3CFGR 0x7CC +#define RCC_SAI4CFGR 0x7D0 +#define RCC_MDF1CFGR 0x7D8 +#define RCC_ADF1CFGR 0x7DC +#define RCC_FDCANCFGR 0x7E0 +#define RCC_HDPCFGR 0x7E4 +#define RCC_ADC12CFGR 0x7E8 +#define RCC_ADC3CFGR 0x7EC +#define RCC_ETH1CFGR 0x7F0 +#define RCC_ETH2CFGR 0x7F4 +#define RCC_USBHCFGR 0x7FC +#define RCC_USB2PHY1CFGR 0x800 +#define RCC_USB2PHY2CFGR 0x804 +#define RCC_USB3DRCFGR 0x808 +#define RCC_USB3PCIEPHYCFGR 0x80C +#define RCC_PCIECFGR 0x810 +#define RCC_USBTCCFGR 0x814 +#define RCC_ETHSWCFGR 0x818 +#define RCC_ETHSWACMCFGR 0x81C +#define RCC_ETHSWACMMSGCFGR 0x820 +#define RCC_STGENCFGR 0x824 +#define RCC_SDMMC1CFGR 0x830 +#define RCC_SDMMC2CFGR 0x834 +#define RCC_SDMMC3CFGR 0x838 +#define RCC_GPUCFGR 0x83C +#define RCC_LTDCCFGR 0x840 +#define RCC_DSICFGR 0x844 +#define RCC_LVDSCFGR 0x850 +#define RCC_CSICFGR 0x858 +#define RCC_DCMIPPCFGR 0x85C +#define RCC_CCICFGR 0x860 +#define RCC_VDECCFGR 0x864 +#define RCC_VENCCFGR 0x868 +#define RCC_RNGCFGR 0x870 +#define RCC_PKACFGR 0x874 +#define RCC_SAESCFGR 0x878 +#define RCC_HASHCFGR 0x87C +#define RCC_CRYP1CFGR 0x880 +#define RCC_CRYP2CFGR 0x884 +#define RCC_IWDG1CFGR 0x888 +#define RCC_IWDG2CFGR 0x88C +#define RCC_IWDG3CFGR 0x890 +#define RCC_IWDG4CFGR 0x894 +#define RCC_IWDG5CFGR 0x898 +#define RCC_WWDG1CFGR 0x89C +#define RCC_WWDG2CFGR 0x8A0 +#define RCC_VREFCFGR 0x8A8 +#define RCC_DTSCFGR 0x8AC +#define RCC_CRCCFGR 0x8B4 +#define RCC_SERCCFGR 0x8B8 +#define RCC_OSPIIOMCFGR 0x8BC +#define RCC_GICV2MCFGR 0x8C0 +#define RCC_I3C1CFGR 0x8C8 +#define RCC_I3C2CFGR 0x8CC +#define RCC_I3C3CFGR 0x8D0 +#define RCC_I3C4CFGR 0x8D4 +#define RCC_MUXSELCFGR 0x1000 +#define RCC_XBAR0CFGR 0x1018 +#define RCC_XBAR1CFGR 0x101C +#define RCC_XBAR2CFGR 0x1020 +#define RCC_XBAR3CFGR 0x1024 +#define RCC_XBAR4CFGR 0x1028 +#define RCC_XBAR5CFGR 0x102C +#define RCC_XBAR6CFGR 0x1030 +#define RCC_XBAR7CFGR 0x1034 +#define RCC_XBAR8CFGR 0x1038 +#define RCC_XBAR9CFGR 0x103C +#define RCC_XBAR10CFGR 0x1040 +#define RCC_XBAR11CFGR 0x1044 +#define RCC_XBAR12CFGR 0x1048 +#define RCC_XBAR13CFGR 0x104C +#define RCC_XBAR14CFGR 0x1050 +#define RCC_XBAR15CFGR 0x1054 +#define RCC_XBAR16CFGR 0x1058 +#define RCC_XBAR17CFGR 0x105C +#define RCC_XBAR18CFGR 0x1060 +#define RCC_XBAR19CFGR 0x1064 +#define RCC_XBAR20CFGR 0x1068 +#define RCC_XBAR21CFGR 0x106C +#define RCC_XBAR22CFGR 0x1070 +#define RCC_XBAR23CFGR 0x1074 +#define RCC_XBAR24CFGR 0x1078 +#define RCC_XBAR25CFGR 0x107C +#define RCC_XBAR26CFGR 0x1080 +#define RCC_XBAR27CFGR 0x1084 +#define RCC_XBAR28CFGR 0x1088 +#define RCC_XBAR29CFGR 0x108C +#define RCC_XBAR30CFGR 0x1090 +#define RCC_XBAR31CFGR 0x1094 +#define RCC_XBAR32CFGR 0x1098 +#define RCC_XBAR33CFGR 0x109C +#define RCC_XBAR34CFGR 0x10A0 +#define RCC_XBAR35CFGR 0x10A4 +#define RCC_XBAR36CFGR 0x10A8 +#define RCC_XBAR37CFGR 0x10AC +#define RCC_XBAR38CFGR 0x10B0 +#define RCC_XBAR39CFGR 0x10B4 +#define RCC_XBAR40CFGR 0x10B8 +#define RCC_XBAR41CFGR 0x10BC +#define RCC_XBAR42CFGR 0x10C0 +#define RCC_XBAR43CFGR 0x10C4 +#define RCC_XBAR44CFGR 0x10C8 +#define RCC_XBAR45CFGR 0x10CC +#define RCC_XBAR46CFGR 0x10D0 +#define RCC_XBAR47CFGR 0x10D4 +#define RCC_XBAR48CFGR 0x10D8 +#define RCC_XBAR49CFGR 0x10DC +#define RCC_XBAR50CFGR 0x10E0 +#define RCC_XBAR51CFGR 0x10E4 +#define RCC_XBAR52CFGR 0x10E8 +#define RCC_XBAR53CFGR 0x10EC +#define RCC_XBAR54CFGR 0x10F0 +#define RCC_XBAR55CFGR 0x10F4 +#define RCC_XBAR56CFGR 0x10F8 +#define RCC_XBAR57CFGR 0x10FC +#define RCC_XBAR58CFGR 0x1100 +#define RCC_XBAR59CFGR 0x1104 +#define RCC_XBAR60CFGR 0x1108 +#define RCC_XBAR61CFGR 0x110C +#define RCC_XBAR62CFGR 0x1110 +#define RCC_XBAR63CFGR 0x1114 +#define RCC_PREDIV0CFGR 0x1118 +#define RCC_PREDIV1CFGR 0x111C +#define RCC_PREDIV2CFGR 0x1120 +#define RCC_PREDIV3CFGR 0x1124 +#define RCC_PREDIV4CFGR 0x1128 +#define RCC_PREDIV5CFGR 0x112C +#define RCC_PREDIV6CFGR 0x1130 +#define RCC_PREDIV7CFGR 0x1134 +#define RCC_PREDIV8CFGR 0x1138 +#define RCC_PREDIV9CFGR 0x113C +#define RCC_PREDIV10CFGR 0x1140 +#define RCC_PREDIV11CFGR 0x1144 +#define RCC_PREDIV12CFGR 0x1148 +#define RCC_PREDIV13CFGR 0x114C +#define RCC_PREDIV14CFGR 0x1150 +#define RCC_PREDIV15CFGR 0x1154 +#define RCC_PREDIV16CFGR 0x1158 +#define RCC_PREDIV17CFGR 0x115C +#define RCC_PREDIV18CFGR 0x1160 +#define RCC_PREDIV19CFGR 0x1164 +#define RCC_PREDIV20CFGR 0x1168 +#define RCC_PREDIV21CFGR 0x116C +#define RCC_PREDIV22CFGR 0x1170 +#define RCC_PREDIV23CFGR 0x1174 +#define RCC_PREDIV24CFGR 0x1178 +#define RCC_PREDIV25CFGR 0x117C +#define RCC_PREDIV26CFGR 0x1180 +#define RCC_PREDIV27CFGR 0x1184 +#define RCC_PREDIV28CFGR 0x1188 +#define RCC_PREDIV29CFGR 0x118C +#define RCC_PREDIV30CFGR 0x1190 +#define RCC_PREDIV31CFGR 0x1194 +#define RCC_PREDIV32CFGR 0x1198 +#define RCC_PREDIV33CFGR 0x119C +#define RCC_PREDIV34CFGR 0x11A0 +#define RCC_PREDIV35CFGR 0x11A4 +#define RCC_PREDIV36CFGR 0x11A8 +#define RCC_PREDIV37CFGR 0x11AC +#define RCC_PREDIV38CFGR 0x11B0 +#define RCC_PREDIV39CFGR 0x11B4 +#define RCC_PREDIV40CFGR 0x11B8 +#define RCC_PREDIV41CFGR 0x11BC +#define RCC_PREDIV42CFGR 0x11C0 +#define RCC_PREDIV43CFGR 0x11C4 +#define RCC_PREDIV44CFGR 0x11C8 +#define RCC_PREDIV45CFGR 0x11CC +#define RCC_PREDIV46CFGR 0x11D0 +#define RCC_PREDIV47CFGR 0x11D4 +#define RCC_PREDIV48CFGR 0x11D8 +#define RCC_PREDIV49CFGR 0x11DC +#define RCC_PREDIV50CFGR 0x11E0 +#define RCC_PREDIV51CFGR 0x11E4 +#define RCC_PREDIV52CFGR 0x11E8 +#define RCC_PREDIV53CFGR 0x11EC +#define RCC_PREDIV54CFGR 0x11F0 +#define RCC_PREDIV55CFGR 0x11F4 +#define RCC_PREDIV56CFGR 0x11F8 +#define RCC_PREDIV57CFGR 0x11FC +#define RCC_PREDIV58CFGR 0x1200 +#define RCC_PREDIV59CFGR 0x1204 +#define RCC_PREDIV60CFGR 0x1208 +#define RCC_PREDIV61CFGR 0x120C +#define RCC_PREDIV62CFGR 0x1210 +#define RCC_PREDIV63CFGR 0x1214 +#define RCC_PREDIVSR1 0x1218 +#define RCC_PREDIVSR2 0x121C +#define RCC_FINDIV0CFGR 0x1224 +#define RCC_FINDIV1CFGR 0x1228 +#define RCC_FINDIV2CFGR 0x122C +#define RCC_FINDIV3CFGR 0x1230 +#define RCC_FINDIV4CFGR 0x1234 +#define RCC_FINDIV5CFGR 0x1238 +#define RCC_FINDIV6CFGR 0x123C +#define RCC_FINDIV7CFGR 0x1240 +#define RCC_FINDIV8CFGR 0x1244 +#define RCC_FINDIV9CFGR 0x1248 +#define RCC_FINDIV10CFGR 0x124C +#define RCC_FINDIV11CFGR 0x1250 +#define RCC_FINDIV12CFGR 0x1254 +#define RCC_FINDIV13CFGR 0x1258 +#define RCC_FINDIV14CFGR 0x125C +#define RCC_FINDIV15CFGR 0x1260 +#define RCC_FINDIV16CFGR 0x1264 +#define RCC_FINDIV17CFGR 0x1268 +#define RCC_FINDIV18CFGR 0x126C +#define RCC_FINDIV19CFGR 0x1270 +#define RCC_FINDIV20CFGR 0x1274 +#define RCC_FINDIV21CFGR 0x1278 +#define RCC_FINDIV22CFGR 0x127C +#define RCC_FINDIV23CFGR 0x1280 +#define RCC_FINDIV24CFGR 0x1284 +#define RCC_FINDIV25CFGR 0x1288 +#define RCC_FINDIV26CFGR 0x128C +#define RCC_FINDIV27CFGR 0x1290 +#define RCC_FINDIV28CFGR 0x1294 +#define RCC_FINDIV29CFGR 0x1298 +#define RCC_FINDIV30CFGR 0x129C +#define RCC_FINDIV31CFGR 0x12A0 +#define RCC_FINDIV32CFGR 0x12A4 +#define RCC_FINDIV33CFGR 0x12A8 +#define RCC_FINDIV34CFGR 0x12AC +#define RCC_FINDIV35CFGR 0x12B0 +#define RCC_FINDIV36CFGR 0x12B4 +#define RCC_FINDIV37CFGR 0x12B8 +#define RCC_FINDIV38CFGR 0x12BC +#define RCC_FINDIV39CFGR 0x12C0 +#define RCC_FINDIV40CFGR 0x12C4 +#define RCC_FINDIV41CFGR 0x12C8 +#define RCC_FINDIV42CFGR 0x12CC +#define RCC_FINDIV43CFGR 0x12D0 +#define RCC_FINDIV44CFGR 0x12D4 +#define RCC_FINDIV45CFGR 0x12D8 +#define RCC_FINDIV46CFGR 0x12DC +#define RCC_FINDIV47CFGR 0x12E0 +#define RCC_FINDIV48CFGR 0x12E4 +#define RCC_FINDIV49CFGR 0x12E8 +#define RCC_FINDIV50CFGR 0x12EC +#define RCC_FINDIV51CFGR 0x12F0 +#define RCC_FINDIV52CFGR 0x12F4 +#define RCC_FINDIV53CFGR 0x12F8 +#define RCC_FINDIV54CFGR 0x12FC +#define RCC_FINDIV55CFGR 0x1300 +#define RCC_FINDIV56CFGR 0x1304 +#define RCC_FINDIV57CFGR 0x1308 +#define RCC_FINDIV58CFGR 0x130C +#define RCC_FINDIV59CFGR 0x1310 +#define RCC_FINDIV60CFGR 0x1314 +#define RCC_FINDIV61CFGR 0x1318 +#define RCC_FINDIV62CFGR 0x131C +#define RCC_FINDIV63CFGR 0x1320 +#define RCC_FINDIVSR1 0x1324 +#define RCC_FINDIVSR2 0x1328 +#define RCC_FCALCOBS0CFGR 0x1340 +#define RCC_FCALCOBS1CFGR 0x1344 +#define RCC_FCALCREFCFGR 0x1348 +#define RCC_FCALCCR1 0x134C +#define RCC_FCALCCR2 0x1354 +#define RCC_FCALCSR 0x1358 +#define RCC_PLL4CFGR1 0x1360 +#define RCC_PLL4CFGR2 0x1364 +#define RCC_PLL4CFGR3 0x1368 +#define RCC_PLL4CFGR4 0x136C +#define RCC_PLL4CFGR5 0x1370 +#define RCC_PLL4CFGR6 0x1378 +#define RCC_PLL4CFGR7 0x137C +#define RCC_PLL5CFGR1 0x1388 +#define RCC_PLL5CFGR2 0x138C +#define RCC_PLL5CFGR3 0x1390 +#define RCC_PLL5CFGR4 0x1394 +#define RCC_PLL5CFGR5 0x1398 +#define RCC_PLL5CFGR6 0x13A0 +#define RCC_PLL5CFGR7 0x13A4 +#define RCC_PLL6CFGR1 0x13B0 +#define RCC_PLL6CFGR2 0x13B4 +#define RCC_PLL6CFGR3 0x13B8 +#define RCC_PLL6CFGR4 0x13BC +#define RCC_PLL6CFGR5 0x13C0 +#define RCC_PLL6CFGR6 0x13C8 +#define RCC_PLL6CFGR7 0x13CC +#define RCC_PLL7CFGR1 0x13D8 +#define RCC_PLL7CFGR2 0x13DC +#define RCC_PLL7CFGR3 0x13E0 +#define RCC_PLL7CFGR4 0x13E4 +#define RCC_PLL7CFGR5 0x13E8 +#define RCC_PLL7CFGR6 0x13F0 +#define RCC_PLL7CFGR7 0x13F4 +#define RCC_PLL8CFGR1 0x1400 +#define RCC_PLL8CFGR2 0x1404 +#define RCC_PLL8CFGR3 0x1408 +#define RCC_PLL8CFGR4 0x140C +#define RCC_PLL8CFGR5 0x1410 +#define RCC_PLL8CFGR6 0x1418 +#define RCC_PLL8CFGR7 0x141C +#define RCC_VERR 0xFFF4 +#define RCC_IDR 0xFFF8 +#define RCC_SIDR 0xFFFC + +#endif /* STM32MP25_RCC_H */ diff --git a/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c b/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c index 9d3ffd3fb2c1..0736f6c81269 100644 --- a/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c @@ -125,6 +125,7 @@ static const struct of_device_id sun20i_d1_r_ccu_ids[] = { { .compatible = "allwinner,sun20i-d1-r-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun20i_d1_r_ccu_ids); static struct platform_driver sun20i_d1_r_ccu_driver = { .probe = sun20i_d1_r_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun20i-d1.c b/drivers/clk/sunxi-ng/ccu-sun20i-d1.c index 48a8fb2c43b7..60756aadfad6 100644 --- a/drivers/clk/sunxi-ng/ccu-sun20i-d1.c +++ b/drivers/clk/sunxi-ng/ccu-sun20i-d1.c @@ -1394,6 +1394,7 @@ static const struct of_device_id sun20i_d1_ccu_ids[] = { { .compatible = "allwinner,sun20i-d1-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun20i_d1_ccu_ids); static struct platform_driver sun20i_d1_ccu_driver = { .probe = sun20i_d1_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c index 451ebb7c99a3..14f5c3da652b 100644 --- a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c +++ b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c @@ -1481,6 +1481,7 @@ static const struct of_device_id sun4i_a10_ccu_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sun4i_a10_ccu_ids); static struct platform_driver sun4i_a10_ccu_driver = { .probe = sun4i_a10_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c index fddd6c877cec..3b983bb59bd9 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c @@ -202,6 +202,7 @@ static const struct of_device_id sun50i_a100_r_ccu_ids[] = { { .compatible = "allwinner,sun50i-a100-r-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_a100_r_ccu_ids); static struct platform_driver sun50i_a100_r_ccu_driver = { .probe = sun50i_a100_r_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100.c b/drivers/clk/sunxi-ng/ccu-sun50i-a100.c index 5f93b5526e13..38aa6e5f298e 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a100.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100.c @@ -1264,6 +1264,7 @@ static const struct of_device_id sun50i_a100_ccu_ids[] = { { .compatible = "allwinner,sun50i-a100-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_a100_ccu_ids); static struct platform_driver sun50i_a100_ccu_driver = { .probe = sun50i_a100_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c index 6a4b2b9ef30a..491b16cfe368 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c @@ -171,11 +171,13 @@ static struct ccu_nkm pll_mipi_clk = { * user manual, and by experiments the PLL doesn't work without * these bits toggled. */ - .enable = BIT(31) | BIT(23) | BIT(22), - .lock = BIT(28), - .n = _SUNXI_CCU_MULT(8, 4), - .k = _SUNXI_CCU_MULT_MIN(4, 2, 2), - .m = _SUNXI_CCU_DIV(0, 4), + .enable = BIT(31) | BIT(23) | BIT(22), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT(8, 4), + .k = _SUNXI_CCU_MULT_MIN(4, 2, 2), + .m = _SUNXI_CCU_DIV(0, 4), + .max_m_n_ratio = 3, + .min_parent_m_ratio = 24000000, .common = { .reg = 0x040, .hw.init = CLK_HW_INIT("pll-mipi", "pll-video0", @@ -980,6 +982,7 @@ static const struct of_device_id sun50i_a64_ccu_ids[] = { { .compatible = "allwinner,sun50i-a64-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_a64_ccu_ids); static struct platform_driver sun50i_a64_ccu_driver = { .probe = sun50i_a64_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c index 02b28cfc5525..e2dc29fa99e7 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c @@ -244,6 +244,7 @@ static const struct of_device_id sun50i_h6_r_ccu_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_h6_r_ccu_ids); static struct platform_driver sun50i_h6_r_ccu_driver = { .probe = sun50i_h6_r_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c index 892df807275c..e6421c2ba122 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c @@ -1274,6 +1274,7 @@ static const struct of_device_id sun50i_h6_ccu_ids[] = { { .compatible = "allwinner,sun50i-h6-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_h6_ccu_ids); static struct platform_driver sun50i_h6_ccu_driver = { .probe = sun50i_h6_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h616.c b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c index 21e918582aa5..45aae1ae5178 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h616.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c @@ -1154,6 +1154,7 @@ static const struct of_device_id sun50i_h616_ccu_ids[] = { { .compatible = "allwinner,sun50i-h616-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_h616_ccu_ids); static struct platform_driver sun50i_h616_ccu_driver = { .probe = sun50i_h616_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c index 0762deffb33c..8cb8cbbdbafb 100644 --- a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c +++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c @@ -1271,6 +1271,7 @@ static const struct of_device_id sun6i_a31_ccu_ids[] = { { .compatible = "allwinner,sun6i-a31-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun6i_a31_ccu_ids); static struct platform_driver sun6i_a31_ccu_driver = { .probe = sun6i_a31_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c index fdc8ccc586c9..5a98c4e9e667 100644 --- a/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c +++ b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c @@ -336,6 +336,7 @@ static const struct of_device_id sun6i_rtc_ccu_match[] = { }, {}, }; +MODULE_DEVICE_TABLE(of, sun6i_rtc_ccu_match); int sun6i_rtc_ccu_probe(struct device *dev, void __iomem *reg) { diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c index e80cc3864e44..e748ad612b8f 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c @@ -751,6 +751,7 @@ static const struct of_device_id sun8i_a23_ccu_ids[] = { { .compatible = "allwinner,sun8i-a23-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_a23_ccu_ids); static struct platform_driver sun8i_a23_ccu_driver = { .probe = sun8i_a23_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c index d12878a1ba9e..8a27a1777600 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c @@ -823,6 +823,7 @@ static const struct of_device_id sun8i_a33_ccu_ids[] = { { .compatible = "allwinner,sun8i-a33-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_a33_ccu_ids); static struct platform_driver sun8i_a33_ccu_driver = { .probe = sun8i_a33_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c index 76cbd9e9e89f..93eca47935cf 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c @@ -911,6 +911,7 @@ static const struct of_device_id sun8i_a83t_ccu_ids[] = { { .compatible = "allwinner,sun8i-a83t-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_a83t_ccu_ids); static struct platform_driver sun8i_a83t_ccu_driver = { .probe = sun8i_a83t_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c index 6a043a0a9dd6..b0b8dba239ae 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c @@ -337,6 +337,7 @@ static const struct of_device_id sunxi_de2_clk_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sunxi_de2_clk_ids); static struct platform_driver sunxi_de2_clk_driver = { .probe = sunxi_de2_clk_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c index 74274c17efb3..ca5739fa04f7 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c @@ -1082,6 +1082,7 @@ static const struct of_device_id sun8i_h3_ccu_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_h3_ccu_ids); static struct platform_driver sun8i_h3_ccu_driver = { .probe = sun8i_h3_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r.c b/drivers/clk/sunxi-ng/ccu-sun8i-r.c index 4890a976b1a0..bac7e737db98 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-r.c @@ -262,6 +262,7 @@ static const struct of_device_id sun8i_r_ccu_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_r_ccu_ids); static struct platform_driver sun8i_r_ccu_driver = { .probe = sun8i_r_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c index 31eca0d3bc1e..3774b293e74c 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c @@ -1363,6 +1363,7 @@ static const struct of_device_id sun8i_r40_ccu_ids[] = { { .compatible = "allwinner,sun8i-r40-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_r40_ccu_ids); static struct platform_driver sun8i_r40_ccu_driver = { .probe = sun8i_r40_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c index f3ce8664b288..994258a3ad2e 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c @@ -768,6 +768,7 @@ static const struct of_device_id sun8i_v3s_ccu_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_v3s_ccu_ids); static struct platform_driver sun8i_v3s_ccu_driver = { .probe = sun8i_v3s_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c index 1d8b1ae1619d..ae7939d3f502 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c @@ -254,6 +254,7 @@ static const struct of_device_id sun9i_a80_de_clk_ids[] = { { .compatible = "allwinner,sun9i-a80-de-clks" }, { } }; +MODULE_DEVICE_TABLE(of, sun9i_a80_de_clk_ids); static struct platform_driver sun9i_a80_de_clk_driver = { .probe = sun9i_a80_de_clk_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c index a0fb0da8f356..bfa2ff9d52a4 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c @@ -127,6 +127,7 @@ static const struct of_device_id sun9i_a80_usb_clk_ids[] = { { .compatible = "allwinner,sun9i-a80-usb-clks" }, { } }; +MODULE_DEVICE_TABLE(of, sun9i_a80_usb_clk_ids); static struct platform_driver sun9i_a80_usb_clk_driver = { .probe = sun9i_a80_usb_clk_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80.c index 730fd8e28014..c05805e4ad22 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80.c @@ -1236,6 +1236,7 @@ static const struct of_device_id sun9i_a80_ccu_ids[] = { { .compatible = "allwinner,sun9i-a80-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun9i_a80_ccu_ids); static struct platform_driver sun9i_a80_ccu_driver = { .probe = sun9i_a80_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c index 0d5b60b123b7..76d3d070b2a7 100644 --- a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c +++ b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c @@ -565,6 +565,7 @@ static const struct of_device_id suniv_f1c100s_ccu_ids[] = { { .compatible = "allwinner,suniv-f1c100s-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, suniv_f1c100s_ccu_ids); static struct platform_driver suniv_f1c100s_ccu_driver = { .probe = suniv_f1c100s_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu_nkm.c b/drivers/clk/sunxi-ng/ccu_nkm.c index 853f84398e2b..1168d894d636 100644 --- a/drivers/clk/sunxi-ng/ccu_nkm.c +++ b/drivers/clk/sunxi-ng/ccu_nkm.c @@ -16,6 +16,20 @@ struct _ccu_nkm { unsigned long m, min_m, max_m; }; +static bool ccu_nkm_is_valid_rate(struct ccu_common *common, unsigned long parent, + unsigned long n, unsigned long m) +{ + struct ccu_nkm *nkm = container_of(common, struct ccu_nkm, common); + + if (nkm->max_m_n_ratio && (m > nkm->max_m_n_ratio * n)) + return false; + + if (nkm->min_parent_m_ratio && (parent < nkm->min_parent_m_ratio * m)) + return false; + + return true; +} + static unsigned long ccu_nkm_find_best_with_parent_adj(struct ccu_common *common, struct clk_hw *parent_hw, unsigned long *parent, unsigned long rate, @@ -31,6 +45,10 @@ static unsigned long ccu_nkm_find_best_with_parent_adj(struct ccu_common *common unsigned long tmp_rate, tmp_parent; tmp_parent = clk_hw_round_rate(parent_hw, rate * _m / (_n * _k)); + + if (!ccu_nkm_is_valid_rate(common, tmp_parent, _n, _m)) + continue; + tmp_rate = tmp_parent * _n * _k / _m; if (ccu_is_better_rate(common, rate, tmp_rate, best_rate) || @@ -64,6 +82,9 @@ static unsigned long ccu_nkm_find_best(unsigned long parent, unsigned long rate, for (_k = nkm->min_k; _k <= nkm->max_k; _k++) { for (_n = nkm->min_n; _n <= nkm->max_n; _n++) { for (_m = nkm->min_m; _m <= nkm->max_m; _m++) { + if (!ccu_nkm_is_valid_rate(common, parent, _n, _m)) + continue; + unsigned long tmp_rate; tmp_rate = parent * _n * _k / _m; diff --git a/drivers/clk/sunxi-ng/ccu_nkm.h b/drivers/clk/sunxi-ng/ccu_nkm.h index 6601defb3f38..c409212ee40e 100644 --- a/drivers/clk/sunxi-ng/ccu_nkm.h +++ b/drivers/clk/sunxi-ng/ccu_nkm.h @@ -27,6 +27,8 @@ struct ccu_nkm { struct ccu_mux_internal mux; unsigned int fixed_post_div; + unsigned long max_m_n_ratio; + unsigned long min_parent_m_ratio; struct ccu_common common; }; diff --git a/drivers/clk/ti/dpll.c b/drivers/clk/ti/dpll.c index 403ec81f561b..3386bd1903df 100644 --- a/drivers/clk/ti/dpll.c +++ b/drivers/clk/ti/dpll.c @@ -34,8 +34,6 @@ static const struct clk_ops dpll_m4xen_ck_ops = { .save_context = &omap3_core_dpll_save_context, .restore_context = &omap3_core_dpll_restore_context, }; -#else -static const struct clk_ops dpll_m4xen_ck_ops = {}; #endif #if defined(CONFIG_ARCH_OMAP3) || defined(CONFIG_ARCH_OMAP4) || \ @@ -95,11 +93,7 @@ static const struct clk_ops omap3_dpll_core_ck_ops = { .recalc_rate = &omap3_dpll_recalc, .round_rate = &omap2_dpll_round_rate, }; -#else -static const struct clk_ops omap3_dpll_core_ck_ops = {}; -#endif -#ifdef CONFIG_ARCH_OMAP3 static const struct clk_ops omap3_dpll_ck_ops = { .enable = &omap3_noncore_dpll_enable, .disable = &omap3_noncore_dpll_disable, @@ -137,9 +131,13 @@ static const struct clk_ops omap3_dpll_per_ck_ops = { }; #endif +#if defined(CONFIG_ARCH_OMAP4) || defined(CONFIG_SOC_OMAP5) || \ + defined(CONFIG_SOC_DRA7XX) || defined(CONFIG_SOC_AM33XX) || \ + defined(CONFIG_SOC_AM43XX) static const struct clk_ops dpll_x2_ck_ops = { .recalc_rate = &omap3_clkoutx2_recalc, }; +#endif /** * _register_dpll - low level registration of a DPLL clock diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c index df3567c1e93b..6c9f0888a2a7 100644 --- a/drivers/cpufreq/pmac32-cpufreq.c +++ b/drivers/cpufreq/pmac32-cpufreq.c @@ -120,9 +120,9 @@ static int cpu_750fx_cpu_speed(int low_speed) /* tweak L2 for high voltage */ if (has_cpu_l2lve) { - hid2 = mfspr(SPRN_HID2); + hid2 = mfspr(SPRN_HID2_750FX); hid2 &= ~0x2000; - mtspr(SPRN_HID2, hid2); + mtspr(SPRN_HID2_750FX, hid2); } } #ifdef CONFIG_PPC_BOOK3S_32 @@ -131,9 +131,9 @@ static int cpu_750fx_cpu_speed(int low_speed) if (low_speed == 1) { /* tweak L2 for low voltage */ if (has_cpu_l2lve) { - hid2 = mfspr(SPRN_HID2); + hid2 = mfspr(SPRN_HID2_750FX); hid2 |= 0x2000; - mtspr(SPRN_HID2, hid2); + mtspr(SPRN_HID2_750FX, hid2); } /* ramping down, set voltage last */ diff --git a/drivers/crypto/intel/qat/qat_420xx/Makefile b/drivers/crypto/intel/qat/qat_420xx/Makefile index a90fbe00b3c8..45728659fbc4 100644 --- a/drivers/crypto/intel/qat/qat_420xx/Makefile +++ b/drivers/crypto/intel/qat/qat_420xx/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx.o qat_420xx-objs := adf_drv.o adf_420xx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_4xxx/Makefile b/drivers/crypto/intel/qat/qat_4xxx/Makefile index ff9c8b5897ea..9ba202079a22 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/Makefile +++ b/drivers/crypto/intel/qat/qat_4xxx/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx.o qat_4xxx-objs := adf_drv.o adf_4xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxx/Makefile b/drivers/crypto/intel/qat/qat_c3xxx/Makefile index 92ef416ccc78..7a06ad519bfc 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/Makefile +++ b/drivers/crypto/intel/qat/qat_c3xxx/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx.o qat_c3xxx-objs := adf_drv.o adf_c3xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile index b6d76825a92c..7ef633058c4f 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf.o qat_c3xxxvf-objs := adf_drv.o adf_c3xxxvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62x/Makefile b/drivers/crypto/intel/qat/qat_c62x/Makefile index d581f7c87d6c..cc9255b3b198 100644 --- a/drivers/crypto/intel/qat/qat_c62x/Makefile +++ b/drivers/crypto/intel/qat/qat_c62x/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x.o qat_c62x-objs := adf_drv.o adf_c62x_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62xvf/Makefile b/drivers/crypto/intel/qat/qat_c62xvf/Makefile index 446c3d638605..256786662d60 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/Makefile +++ b/drivers/crypto/intel/qat/qat_c62xvf/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf.o qat_c62xvf-objs := adf_drv.o adf_c62xvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index 9da2278bd5b7..04260f61d042 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -130,8 +130,7 @@ static void adf_device_reset_worker(struct work_struct *work) if (adf_dev_restart(accel_dev)) { /* The device hanged and we can't restart it so stop here */ dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); - if (reset_data->mode == ADF_DEV_RESET_ASYNC || - completion_done(&reset_data->compl)) + if (reset_data->mode == ADF_DEV_RESET_ASYNC) kfree(reset_data); WARN(1, "QAT: device restart failed. Device is unusable\n"); return; @@ -147,16 +146,8 @@ static void adf_device_reset_worker(struct work_struct *work) adf_dev_restarted_notify(accel_dev); clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); - /* - * The dev is back alive. Notify the caller if in sync mode - * - * If device restart will take a more time than expected, - * the schedule_reset() function can timeout and exit. This can be - * detected by calling the completion_done() function. In this case - * the reset_data structure needs to be freed here. - */ - if (reset_data->mode == ADF_DEV_RESET_ASYNC || - completion_done(&reset_data->compl)) + /* The dev is back alive. Notify the caller if in sync mode */ + if (reset_data->mode == ADF_DEV_RESET_ASYNC) kfree(reset_data); else complete(&reset_data->compl); @@ -191,10 +182,10 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, if (!timeout) { dev_err(&GET_DEV(accel_dev), "Reset device timeout expired\n"); + cancel_work_sync(&reset_data->reset_work); ret = -EFAULT; - } else { - kfree(reset_data); } + kfree(reset_data); return ret; } return 0; diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile index 38d6f8e1624a..cfd3bd757715 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile +++ b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o qat_dh895xcc-objs := adf_drv.o adf_dh895xcc_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile index 0153c85ce743..64b54e92b2b4 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf.o qat_dh895xccvf-objs := adf_drv.o adf_dh895xccvf_hw_data.o diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 0df09bd79408..8567dd11eaac 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -525,7 +525,7 @@ static int cxl_cdat_get_length(struct device *dev, __le32 response[2]; int rc; - rc = pci_doe(doe_mb, PCI_DVSEC_VENDOR_ID_CXL, + rc = pci_doe(doe_mb, PCI_VENDOR_ID_CXL, CXL_DOE_PROTOCOL_TABLE_ACCESS, &request, sizeof(request), &response, sizeof(response)); @@ -555,7 +555,7 @@ static int cxl_cdat_read_table(struct device *dev, __le32 request = CDAT_DOE_REQ(entry_handle); int rc; - rc = pci_doe(doe_mb, PCI_DVSEC_VENDOR_ID_CXL, + rc = pci_doe(doe_mb, PCI_VENDOR_ID_CXL, CXL_DOE_PROTOCOL_TABLE_ACCESS, &request, sizeof(request), rsp, sizeof(*rsp) + remaining); @@ -640,7 +640,7 @@ void read_cdat_data(struct cxl_port *port) if (!pdev) return; - doe_mb = pci_find_doe_mailbox(pdev, PCI_DVSEC_VENDOR_ID_CXL, + doe_mb = pci_find_doe_mailbox(pdev, PCI_VENDOR_ID_CXL, CXL_DOE_PROTOCOL_TABLE_ACCESS); if (!doe_mb) { dev_dbg(dev, "No CDAT mailbox\n"); @@ -1045,3 +1045,32 @@ long cxl_pci_get_latency(struct pci_dev *pdev) return cxl_flit_size(pdev) * MEGA / bw; } + +static int __cxl_endpoint_decoder_reset_detected(struct device *dev, void *data) +{ + struct cxl_port *port = data; + struct cxl_decoder *cxld; + struct cxl_hdm *cxlhdm; + void __iomem *hdm; + u32 ctrl; + + if (!is_endpoint_decoder(dev)) + return 0; + + cxld = to_cxl_decoder(dev); + if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0) + return 0; + + cxlhdm = dev_get_drvdata(&port->dev); + hdm = cxlhdm->regs.hdm_decoder; + ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id)); + + return !FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl); +} + +bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port) +{ + return device_for_each_child(&port->dev, port, + __cxl_endpoint_decoder_reset_detected); +} +EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL); diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c index 3c42f984eeaf..e1082e749c69 100644 --- a/drivers/cxl/core/regs.c +++ b/drivers/cxl/core/regs.c @@ -314,7 +314,7 @@ int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type, .resource = CXL_RESOURCE_NONE, }; - regloc = pci_find_dvsec_capability(pdev, PCI_DVSEC_VENDOR_ID_CXL, + regloc = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, CXL_DVSEC_REG_LOCATOR); if (!regloc) return -ENXIO; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 80f58b96dc1c..603c0120cff8 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -898,6 +898,8 @@ void cxl_coordinates_combine(struct access_coordinate *out, struct access_coordinate *c1, struct access_coordinate *c2); +bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port); + /* * Unit test builds overrides this to __weak, find the 'strong' version * of these symbols in tools/testing/cxl/. diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h index 93992a1c8eec..4da07727ab9c 100644 --- a/drivers/cxl/cxlpci.h +++ b/drivers/cxl/cxlpci.h @@ -13,7 +13,6 @@ * "DVSEC" redundancies removed. When obvious, abbreviations may be used. */ #define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20) -#define PCI_DVSEC_VENDOR_ID_CXL 0x1E98 /* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */ #define CXL_DVSEC_PCIE_DEVICE 0 diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 74876c9835e8..e53646e9f2fb 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -817,7 +817,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) cxlds->rcd = is_cxl_restricted(pdev); cxlds->serial = pci_get_dsn(pdev); cxlds->cxl_dvsec = pci_find_dvsec_capability( - pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE); + pdev, PCI_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE); if (!cxlds->cxl_dvsec) dev_warn(&pdev->dev, "Device DVSEC not present, skip CXL.mem init\n"); @@ -957,11 +957,33 @@ static void cxl_error_resume(struct pci_dev *pdev) dev->driver ? "successful" : "failed"); } +static void cxl_reset_done(struct pci_dev *pdev) +{ + struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); + struct cxl_memdev *cxlmd = cxlds->cxlmd; + struct device *dev = &pdev->dev; + + /* + * FLR does not expect to touch the HDM decoders and related + * registers. SBR, however, will wipe all device configurations. + * Issue a warning if there was an active decoder before the reset + * that no longer exists. + */ + guard(device)(&cxlmd->dev); + if (cxlmd->endpoint && + cxl_endpoint_decoder_reset_detected(cxlmd->endpoint)) { + dev_crit(dev, "SBR happened without memory regions removal.\n"); + dev_crit(dev, "System may be unstable if regions hosted system memory.\n"); + add_taint(TAINT_USER, LOCKDEP_STILL_OK); + } +} + static const struct pci_error_handlers cxl_error_handlers = { .error_detected = cxl_error_detected, .slot_reset = cxl_slot_reset, .resume = cxl_error_resume, .cor_error_detected = cxl_cor_error_detected, + .reset_done = cxl_reset_done, }; static struct pci_driver cxl_pci_driver = { diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 6ec5db8013e8..3ef9550bd2ca 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -192,7 +192,7 @@ static u64 dev_dax_size(struct dev_dax *dev_dax) u64 size = 0; int i; - WARN_ON_ONCE(!rwsem_is_locked(&dax_dev_rwsem)); + lockdep_assert_held(&dax_dev_rwsem); for (i = 0; i < dev_dax->nr_range; i++) size += range_len(&dev_dax->ranges[i].range); @@ -302,7 +302,7 @@ static unsigned long long dax_region_avail_size(struct dax_region *dax_region) resource_size_t size = resource_size(&dax_region->res); struct resource *res; - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held(&dax_region_rwsem); for_each_dax_region_resource(dax_region, res) size -= resource_size(res); @@ -447,7 +447,7 @@ static void trim_dev_dax_range(struct dev_dax *dev_dax) struct range *range = &dev_dax->ranges[i].range; struct dax_region *dax_region = dev_dax->region; - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held_write(&dax_region_rwsem); dev_dbg(&dev_dax->dev, "delete range[%d]: %#llx:%#llx\n", i, (unsigned long long)range->start, (unsigned long long)range->end); @@ -465,26 +465,17 @@ static void free_dev_dax_ranges(struct dev_dax *dev_dax) trim_dev_dax_range(dev_dax); } -static void __unregister_dev_dax(void *dev) +static void unregister_dev_dax(void *dev) { struct dev_dax *dev_dax = to_dev_dax(dev); dev_dbg(dev, "%s\n", __func__); + down_write(&dax_region_rwsem); kill_dev_dax(dev_dax); device_del(dev); free_dev_dax_ranges(dev_dax); put_device(dev); -} - -static void unregister_dev_dax(void *dev) -{ - if (rwsem_is_locked(&dax_region_rwsem)) - return __unregister_dev_dax(dev); - - if (WARN_ON_ONCE(down_write_killable(&dax_region_rwsem) != 0)) - return; - __unregister_dev_dax(dev); up_write(&dax_region_rwsem); } @@ -507,7 +498,7 @@ static int __free_dev_dax_id(struct dev_dax *dev_dax) struct dax_region *dax_region; int rc = dev_dax->id; - WARN_ON_ONCE(!rwsem_is_locked(&dax_dev_rwsem)); + lockdep_assert_held_write(&dax_dev_rwsem); if (!dev_dax->dyn_id || dev_dax->id < 0) return -1; @@ -560,15 +551,10 @@ static ssize_t delete_store(struct device *dev, struct device_attribute *attr, if (!victim) return -ENXIO; - rc = down_write_killable(&dax_region_rwsem); - if (rc) - return rc; - rc = down_write_killable(&dax_dev_rwsem); - if (rc) { - up_write(&dax_region_rwsem); - return rc; - } + device_lock(dev); + device_lock(victim); dev_dax = to_dev_dax(victim); + down_write(&dax_dev_rwsem); if (victim->driver || dev_dax_size(dev_dax)) rc = -EBUSY; else { @@ -589,11 +575,12 @@ static ssize_t delete_store(struct device *dev, struct device_attribute *attr, rc = -EBUSY; } up_write(&dax_dev_rwsem); + device_unlock(victim); /* won the race to invalidate the device, clean it up */ if (do_del) devm_release_action(dev, unregister_dev_dax, victim); - up_write(&dax_region_rwsem); + device_unlock(dev); put_device(victim); return rc; @@ -705,7 +692,7 @@ static void dax_mapping_release(struct device *dev) put_device(parent); } -static void __unregister_dax_mapping(void *data) +static void unregister_dax_mapping(void *data) { struct device *dev = data; struct dax_mapping *mapping = to_dax_mapping(dev); @@ -713,25 +700,12 @@ static void __unregister_dax_mapping(void *data) dev_dbg(dev, "%s\n", __func__); - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); - dev_dax->ranges[mapping->range_id].mapping = NULL; mapping->range_id = -1; device_unregister(dev); } -static void unregister_dax_mapping(void *data) -{ - if (rwsem_is_locked(&dax_region_rwsem)) - return __unregister_dax_mapping(data); - - if (WARN_ON_ONCE(down_write_killable(&dax_region_rwsem) != 0)) - return; - __unregister_dax_mapping(data); - up_write(&dax_region_rwsem); -} - static struct dev_dax_range *get_dax_range(struct device *dev) { struct dax_mapping *mapping = to_dax_mapping(dev); @@ -830,7 +804,7 @@ static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id) struct device *dev; int rc; - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held_write(&dax_region_rwsem); if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver, "region disabled\n")) @@ -876,7 +850,7 @@ static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start, struct resource *alloc; int i, rc; - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held_write(&dax_region_rwsem); /* handle the seed alloc special case */ if (!size) { @@ -935,7 +909,7 @@ static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, r struct device *dev = &dev_dax->dev; int rc; - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held_write(&dax_region_rwsem); if (dev_WARN_ONCE(dev, !size, "deletion is handled by dev_dax_shrink\n")) return -EINVAL; @@ -963,11 +937,11 @@ static ssize_t size_show(struct device *dev, unsigned long long size; int rc; - rc = down_write_killable(&dax_dev_rwsem); + rc = down_read_interruptible(&dax_dev_rwsem); if (rc) return rc; size = dev_dax_size(dev_dax); - up_write(&dax_dev_rwsem); + up_read(&dax_dev_rwsem); return sysfs_emit(buf, "%llu\n", size); } @@ -1565,12 +1539,8 @@ err_id: struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data) { struct dev_dax *dev_dax; - int rc; - - rc = down_write_killable(&dax_region_rwsem); - if (rc) - return ERR_PTR(rc); + down_write(&dax_region_rwsem); dev_dax = __devm_create_dev_dax(data); up_write(&dax_region_rwsem); diff --git a/drivers/dax/device.c b/drivers/dax/device.c index c24ef4d3cf31..eb61598247a9 100644 --- a/drivers/dax/device.c +++ b/drivers/dax/device.c @@ -329,14 +329,14 @@ static unsigned long dax_get_unmapped_area(struct file *filp, if ((off + len_align) < off) goto out; - addr_align = current->mm->get_unmapped_area(filp, addr, len_align, - pgoff, flags); + addr_align = mm_get_unmapped_area(current->mm, filp, addr, len_align, + pgoff, flags); if (!IS_ERR_VALUE(addr_align)) { addr_align += (off - addr_align) & (align - 1); return addr_align; } out: - return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, filp, addr, len, pgoff, flags); } static const struct address_space_operations dev_dax_aops = { diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 42ee360cf4e3..4fe9d040e375 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -55,36 +55,14 @@ static LIST_HEAD(kmem_memory_types); static struct memory_dev_type *kmem_find_alloc_memory_type(int adist) { - bool found = false; - struct memory_dev_type *mtype; - - mutex_lock(&kmem_memory_type_lock); - list_for_each_entry(mtype, &kmem_memory_types, list) { - if (mtype->adistance == adist) { - found = true; - break; - } - } - if (!found) { - mtype = alloc_memory_type(adist); - if (!IS_ERR(mtype)) - list_add(&mtype->list, &kmem_memory_types); - } - mutex_unlock(&kmem_memory_type_lock); - - return mtype; + guard(mutex)(&kmem_memory_type_lock); + return mt_find_alloc_memory_type(adist, &kmem_memory_types); } static void kmem_put_memory_types(void) { - struct memory_dev_type *mtype, *mtn; - - mutex_lock(&kmem_memory_type_lock); - list_for_each_entry_safe(mtype, mtn, &kmem_memory_types, list) { - list_del(&mtype->list); - put_memory_type(mtype); - } - mutex_unlock(&kmem_memory_type_lock); + guard(mutex)(&kmem_memory_type_lock); + mt_put_memory_types(&kmem_memory_types); } static int dev_dax_kmem_probe(struct dev_dax *dev_dax) diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index dfd40d14e408..802ca916f05f 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -31,10 +31,12 @@ obj-$(CONFIG_DW_AXI_DMAC) += dw-axi-dmac/ obj-$(CONFIG_DW_DMAC_CORE) += dw/ obj-$(CONFIG_DW_EDMA) += dw-edma/ obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o +fsl-edma-trace-$(CONFIG_TRACING) := fsl-edma-trace.o +CFLAGS_fsl-edma-trace.o := -I$(src) obj-$(CONFIG_FSL_DMA) += fsldma.o -fsl-edma-objs := fsl-edma-main.o fsl-edma-common.o +fsl-edma-objs := fsl-edma-main.o fsl-edma-common.o ${fsl-edma-trace-y} obj-$(CONFIG_FSL_EDMA) += fsl-edma.o -mcf-edma-objs := mcf-edma-main.o fsl-edma-common.o +mcf-edma-objs := mcf-edma-main.o fsl-edma-common.o ${fsl-edma-trace-y} obj-$(CONFIG_MCF_EDMA) += mcf-edma.o obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o obj-$(CONFIG_FSL_RAID) += fsl_raid.o diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index fbf048f432bf..73a5cfb4da8a 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -2855,8 +2855,8 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) } /* Initialize physical channels */ - pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)), - GFP_KERNEL); + pl08x->phy_chans = kcalloc(vd->channels, sizeof(*pl08x->phy_chans), + GFP_KERNEL); if (!pl08x->phy_chans) { ret = -ENOMEM; goto out_no_phychans; diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 4e339c04fc1e..bdb752f11869 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -1002,6 +1002,16 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) return 0; } +static void axi_dmac_tasklet_kill(void *task) +{ + tasklet_kill(task); +} + +static void axi_dmac_free_dma_controller(void *of_node) +{ + of_dma_controller_free(of_node); +} + static int axi_dmac_probe(struct platform_device *pdev) { struct dma_device *dma_dev; @@ -1025,14 +1035,10 @@ static int axi_dmac_probe(struct platform_device *pdev) if (IS_ERR(dmac->base)) return PTR_ERR(dmac->base); - dmac->clk = devm_clk_get(&pdev->dev, NULL); + dmac->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(dmac->clk)) return PTR_ERR(dmac->clk); - ret = clk_prepare_enable(dmac->clk); - if (ret < 0) - return ret; - version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION); if (version >= ADI_AXI_PCORE_VER(4, 3, 'a')) @@ -1041,7 +1047,7 @@ static int axi_dmac_probe(struct platform_device *pdev) ret = axi_dmac_parse_dt(&pdev->dev, dmac); if (ret < 0) - goto err_clk_disable; + return ret; INIT_LIST_HEAD(&dmac->chan.active_descs); @@ -1072,7 +1078,7 @@ static int axi_dmac_probe(struct platform_device *pdev) ret = axi_dmac_detect_caps(dmac, version); if (ret) - goto err_clk_disable; + return ret; dma_dev->copy_align = (dmac->chan.address_align_mask + 1); @@ -1088,57 +1094,42 @@ static int axi_dmac_probe(struct platform_device *pdev) !AXI_DMAC_DST_COHERENT_GET(ret)) { dev_err(dmac->dma_dev.dev, "Coherent DMA not supported in hardware"); - ret = -EINVAL; - goto err_clk_disable; + return -EINVAL; } } - ret = dma_async_device_register(dma_dev); + ret = dmaenginem_async_device_register(dma_dev); + if (ret) + return ret; + + /* + * Put the action in here so it get's done before unregistering the DMA + * device. + */ + ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_tasklet_kill, + &dmac->chan.vchan.task); if (ret) - goto err_clk_disable; + return ret; ret = of_dma_controller_register(pdev->dev.of_node, of_dma_xlate_by_chan_id, dma_dev); if (ret) - goto err_unregister_device; + return ret; - ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, IRQF_SHARED, - dev_name(&pdev->dev), dmac); + ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_free_dma_controller, + pdev->dev.of_node); if (ret) - goto err_unregister_of; + return ret; - platform_set_drvdata(pdev, dmac); + ret = devm_request_irq(&pdev->dev, dmac->irq, axi_dmac_interrupt_handler, + IRQF_SHARED, dev_name(&pdev->dev), dmac); + if (ret) + return ret; regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base, &axi_dmac_regmap_config); - if (IS_ERR(regmap)) { - ret = PTR_ERR(regmap); - goto err_free_irq; - } - - return 0; - -err_free_irq: - free_irq(dmac->irq, dmac); -err_unregister_of: - of_dma_controller_free(pdev->dev.of_node); -err_unregister_device: - dma_async_device_unregister(&dmac->dma_dev); -err_clk_disable: - clk_disable_unprepare(dmac->clk); - - return ret; -} - -static void axi_dmac_remove(struct platform_device *pdev) -{ - struct axi_dmac *dmac = platform_get_drvdata(pdev); - of_dma_controller_free(pdev->dev.of_node); - free_irq(dmac->irq, dmac); - tasklet_kill(&dmac->chan.vchan.task); - dma_async_device_unregister(&dmac->dma_dev); - clk_disable_unprepare(dmac->clk); + return PTR_ERR_OR_ZERO(regmap); } static const struct of_device_id axi_dmac_of_match_table[] = { @@ -1153,7 +1144,6 @@ static struct platform_driver axi_dmac_driver = { .of_match_table = axi_dmac_of_match_table, }, .probe = axi_dmac_probe, - .remove_new = axi_dmac_remove, }; module_platform_driver(axi_dmac_driver); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index a86a81ff0caa..fffafa86d964 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -302,6 +302,7 @@ static struct axi_dma_desc *axi_desc_alloc(u32 num) kfree(desc); return NULL; } + desc->nr_hw_descs = num; return desc; } @@ -328,7 +329,7 @@ static struct axi_dma_lli *axi_desc_get(struct axi_dma_chan *chan, static void axi_desc_put(struct axi_dma_desc *desc) { struct axi_dma_chan *chan = desc->chan; - int count = atomic_read(&chan->descs_allocated); + int count = desc->nr_hw_descs; struct axi_dma_hw_desc *hw_desc; int descs_put; @@ -1139,9 +1140,6 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) /* Remove the completed descriptor from issued list before completing */ list_del(&vd->node); vchan_cookie_complete(vd); - - /* Submit queued descriptors after processing the completed ones */ - axi_chan_start_first_queued(chan); } out: @@ -1445,6 +1443,24 @@ static int parse_device_properties(struct axi_dma_chip *chip) return 0; } +static int axi_req_irqs(struct platform_device *pdev, struct axi_dma_chip *chip) +{ + int irq_count = platform_irq_count(pdev); + int ret; + + for (int i = 0; i < irq_count; i++) { + chip->irq[i] = platform_get_irq(pdev, i); + if (chip->irq[i] < 0) + return chip->irq[i]; + ret = devm_request_irq(chip->dev, chip->irq[i], dw_axi_dma_interrupt, + IRQF_SHARED, KBUILD_MODNAME, chip); + if (ret < 0) + return ret; + } + + return 0; +} + static int dw_probe(struct platform_device *pdev) { struct axi_dma_chip *chip; @@ -1471,10 +1487,6 @@ static int dw_probe(struct platform_device *pdev) chip->dev = &pdev->dev; chip->dw->hdata = hdata; - chip->irq = platform_get_irq(pdev, 0); - if (chip->irq < 0) - return chip->irq; - chip->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(chip->regs)) return PTR_ERR(chip->regs); @@ -1515,8 +1527,7 @@ static int dw_probe(struct platform_device *pdev) if (!dw->chan) return -ENOMEM; - ret = devm_request_irq(chip->dev, chip->irq, dw_axi_dma_interrupt, - IRQF_SHARED, KBUILD_MODNAME, chip); + ret = axi_req_irqs(pdev, chip); if (ret) return ret; @@ -1629,7 +1640,9 @@ static void dw_remove(struct platform_device *pdev) pm_runtime_disable(chip->dev); axi_dma_suspend(chip); - devm_free_irq(chip->dev, chip->irq, chip); + for (i = 0; i < DMAC_MAX_CHANNELS; i++) + if (chip->irq[i] > 0) + devm_free_irq(chip->dev, chip->irq[i], chip); of_dma_controller_free(chip->dev->of_node); @@ -1653,6 +1666,9 @@ static const struct of_device_id dw_dma_of_id_table[] = { }, { .compatible = "starfive,jh7110-axi-dma", .data = (void *)(AXI_DMA_FLAG_HAS_RESETS | AXI_DMA_FLAG_USE_CFG2), + }, { + .compatible = "starfive,jh8100-axi-dma", + .data = (void *)AXI_DMA_FLAG_HAS_RESETS, }, {} }; diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index 454904d99654..b842e6a8d90d 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -65,7 +65,7 @@ struct dw_axi_dma { struct axi_dma_chip { struct device *dev; - int irq; + int irq[DMAC_MAX_CHANNELS]; void __iomem *regs; void __iomem *apb_regs; struct clk *core_clk; @@ -104,6 +104,7 @@ struct axi_dma_desc { u32 completed_blocks; u32 length; u32 period_len; + u32 nr_hw_descs; }; struct axi_dma_chan_config { diff --git a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c index 5a8061a307cd..36384d019263 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c +++ b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c @@ -362,7 +362,7 @@ static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) for (i = 0; i < priv->num_pairs; i++) { err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, - i, &priv->rx_queue_attr[i]); + i, 0, &priv->rx_queue_attr[i]); if (err) { dev_err(dev, "dpdmai_get_rx_queue() failed\n"); goto exit; @@ -370,13 +370,13 @@ static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, - i, &priv->tx_fqid[i]); + i, 0, &priv->tx_queue_attr[i]); if (err) { dev_err(dev, "dpdmai_get_tx_queue() failed\n"); goto exit; } - ppriv->req_fqid = priv->tx_fqid[i]; - ppriv->prio = i; + ppriv->req_fqid = priv->tx_queue_attr[i].fqid; + ppriv->prio = DPAA2_QDMA_DEFAULT_PRIORITY; ppriv->priv = priv; ppriv++; } @@ -542,7 +542,7 @@ static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv) rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; rx_queue_cfg.dest_cfg.priority = ppriv->prio; err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, - rx_queue_cfg.dest_cfg.priority, + rx_queue_cfg.dest_cfg.priority, 0, &rx_queue_cfg); if (err) { dev_err(dev, "dpdmai_set_rx_queue() failed\n"); @@ -642,7 +642,7 @@ static int dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine *dpaa2_qdma) for (i = 0; i < dpaa2_qdma->n_chans; i++) { dpaa2_chan = &dpaa2_qdma->chans[i]; dpaa2_chan->qdma = dpaa2_qdma; - dpaa2_chan->fqid = priv->tx_fqid[i % num]; + dpaa2_chan->fqid = priv->tx_queue_attr[i % num].fqid; dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc; vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev); spin_lock_init(&dpaa2_chan->queue_lock); @@ -802,7 +802,7 @@ static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev) dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); dpaa2_dpdmai_dpio_unbind(priv); dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); - dpdmai_destroy(priv->mc_io, 0, ls_dev->mc_handle); + dpdmai_destroy(priv->mc_io, 0, priv->dpqdma_id, ls_dev->mc_handle); } static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { diff --git a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h index 03e2f4e0baca..2c80077cb7c0 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h +++ b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h @@ -6,6 +6,7 @@ #define DPAA2_QDMA_STORE_SIZE 16 #define NUM_CH 8 +#define DPAA2_QDMA_DEFAULT_PRIORITY 0 struct dpaa2_qdma_sd_d { u32 rsv:32; @@ -122,8 +123,8 @@ struct dpaa2_qdma_priv { struct dpaa2_qdma_engine *dpaa2_qdma; struct dpaa2_qdma_priv_per_prio *ppriv; - struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_PRIO_NUM]; - u32 tx_fqid[DPDMAI_PRIO_NUM]; + struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_MAX_QUEUE_NUM]; + struct dpdmai_tx_queue_attr tx_queue_attr[DPDMAI_MAX_QUEUE_NUM]; }; struct dpaa2_qdma_priv_per_prio { diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c index 878662aaa1c2..36897b41ee7e 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c @@ -1,47 +1,52 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright 2019 NXP +#include <linux/bitfield.h> #include <linux/module.h> #include <linux/types.h> #include <linux/io.h> #include <linux/fsl/mc.h> #include "dpdmai.h" +#define DEST_TYPE_MASK 0xF + struct dpdmai_rsp_get_attributes { __le32 id; u8 num_of_priorities; - u8 pad0[3]; + u8 num_of_queues; + u8 pad0[2]; __le16 major; __le16 minor; }; struct dpdmai_cmd_queue { __le32 dest_id; - u8 priority; - u8 queue; + u8 dest_priority; + union { + u8 queue; + u8 pri; + }; u8 dest_type; - u8 pad; + u8 queue_idx; __le64 user_ctx; union { __le32 options; __le32 fqid; }; -}; +} __packed; struct dpdmai_rsp_get_tx_queue { __le64 pad; __le32 fqid; }; -#define MC_CMD_OP(_cmd, _param, _offset, _width, _type, _arg) \ - ((_cmd).params[_param] |= mc_enc((_offset), (_width), _arg)) +struct dpdmai_cmd_open { + __le32 dpdmai_id; +} __packed; -/* cmd, param, offset, width, type, arg_name */ -#define DPDMAI_CMD_CREATE(cmd, cfg) \ -do { \ - MC_CMD_OP(cmd, 0, 8, 8, u8, (cfg)->priorities[0]);\ - MC_CMD_OP(cmd, 0, 16, 8, u8, (cfg)->priorities[1]);\ -} while (0) +struct dpdmai_cmd_destroy { + __le32 dpdmai_id; +} __packed; static inline u64 mc_enc(int lsoffset, int width, u64 val) { @@ -68,16 +73,16 @@ static inline u64 mc_enc(int lsoffset, int width, u64 val) int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpdmai_id, u16 *token) { + struct dpdmai_cmd_open *cmd_params; struct fsl_mc_command cmd = { 0 }; - __le64 *cmd_dpdmai_id; int err; /* prepare command */ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN, cmd_flags, 0); - cmd_dpdmai_id = cmd.params; - *cmd_dpdmai_id = cpu_to_le32(dpdmai_id); + cmd_params = (struct dpdmai_cmd_open *)&cmd.params; + cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id); /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -116,65 +121,26 @@ int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) EXPORT_SYMBOL_GPL(dpdmai_close); /** - * dpdmai_create() - Create the DPDMAI object - * @mc_io: Pointer to MC portal's I/O object - * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' - * @cfg: Configuration structure - * @token: Returned token; use in subsequent API calls - * - * Create the DPDMAI object, allocate required resources and - * perform required initialization. - * - * The object can be created either by declaring it in the - * DPL file, or by calling this function. - * - * This function returns a unique authentication token, - * associated with the specific object ID and the specific MC - * portal; this token must be used in all subsequent calls to - * this specific object. For objects that are created using the - * DPL file, call dpdmai_open() function to get an authentication - * token first. - * - * Return: '0' on Success; Error code otherwise. - */ -int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, - const struct dpdmai_cfg *cfg, u16 *token) -{ - struct fsl_mc_command cmd = { 0 }; - int err; - - /* prepare command */ - cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE, - cmd_flags, 0); - DPDMAI_CMD_CREATE(cmd, cfg); - - /* send command to mc*/ - err = mc_send_command(mc_io, &cmd); - if (err) - return err; - - /* retrieve response parameters */ - *token = mc_cmd_hdr_read_token(&cmd); - - return 0; -} - -/** * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources. * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpdmai_id: The object id; it must be a valid id within the container that created this object; * @token: Token of DPDMAI object * * Return: '0' on Success; error code otherwise. */ -int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u32 dpdmai_id, u16 token) { + struct dpdmai_cmd_destroy *cmd_params; struct fsl_mc_command cmd = { 0 }; /* prepare command */ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY, cmd_flags, token); + cmd_params = (struct dpdmai_cmd_destroy *)&cmd.params; + cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id); + /* send command to mc*/ return mc_send_command(mc_io, &cmd); } @@ -274,6 +240,7 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, attr->version.major = le16_to_cpu(rsp_params->major); attr->version.minor = le16_to_cpu(rsp_params->minor); attr->num_of_priorities = rsp_params->num_of_priorities; + attr->num_of_queues = rsp_params->num_of_queues; return 0; } @@ -284,13 +251,14 @@ EXPORT_SYMBOL_GPL(dpdmai_get_attributes); * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: DMA queue index * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation * @cfg: Rx queue configuration * * Return: '0' on Success; Error code otherwise. */ -int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 queue_idx, u8 priority, const struct dpdmai_rx_queue_cfg *cfg) { struct dpdmai_cmd_queue *cmd_params; @@ -302,11 +270,12 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, cmd_params = (struct dpdmai_cmd_queue *)cmd.params; cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); - cmd_params->priority = cfg->dest_cfg.priority; - cmd_params->queue = priority; + cmd_params->dest_priority = cfg->dest_cfg.priority; + cmd_params->pri = priority; cmd_params->dest_type = cfg->dest_cfg.dest_type; cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); cmd_params->options = cpu_to_le32(cfg->options); + cmd_params->queue_idx = queue_idx; /* send command to mc*/ return mc_send_command(mc_io, &cmd); @@ -318,13 +287,14 @@ EXPORT_SYMBOL_GPL(dpdmai_set_rx_queue); * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: DMA Queue index * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation * @attr: Returned Rx queue attributes * * Return: '0' on Success; Error code otherwise. */ -int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 queue_idx, u8 priority, struct dpdmai_rx_queue_attr *attr) { struct dpdmai_cmd_queue *cmd_params; @@ -337,6 +307,7 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, cmd_params = (struct dpdmai_cmd_queue *)cmd.params; cmd_params->queue = priority; + cmd_params->queue_idx = queue_idx; /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -345,8 +316,8 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, /* retrieve response parameters */ attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id); - attr->dest_cfg.priority = cmd_params->priority; - attr->dest_cfg.dest_type = cmd_params->dest_type; + attr->dest_cfg.priority = cmd_params->dest_priority; + attr->dest_cfg.dest_type = FIELD_GET(DEST_TYPE_MASK, cmd_params->dest_type); attr->user_ctx = le64_to_cpu(cmd_params->user_ctx); attr->fqid = le32_to_cpu(cmd_params->fqid); @@ -359,14 +330,15 @@ EXPORT_SYMBOL_GPL(dpdmai_get_rx_queue); * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: DMA queue index * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation - * @fqid: Returned Tx queue + * @attr: Returned DMA Tx queue attributes * * Return: '0' on Success; Error code otherwise. */ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, - u16 token, u8 priority, u32 *fqid) + u16 token, u8 queue_idx, u8 priority, struct dpdmai_tx_queue_attr *attr) { struct dpdmai_rsp_get_tx_queue *rsp_params; struct dpdmai_cmd_queue *cmd_params; @@ -379,6 +351,7 @@ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, cmd_params = (struct dpdmai_cmd_queue *)cmd.params; cmd_params->queue = priority; + cmd_params->queue_idx = queue_idx; /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -388,7 +361,7 @@ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, /* retrieve response parameters */ rsp_params = (struct dpdmai_rsp_get_tx_queue *)cmd.params; - *fqid = le32_to_cpu(rsp_params->fqid); + attr->fqid = le32_to_cpu(rsp_params->fqid); return 0; } diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h index b13b9bf0c003..3fe7d8327366 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h @@ -5,14 +5,19 @@ #define __FSL_DPDMAI_H /* DPDMAI Version */ -#define DPDMAI_VER_MAJOR 2 -#define DPDMAI_VER_MINOR 2 +#define DPDMAI_VER_MAJOR 3 +#define DPDMAI_VER_MINOR 3 -#define DPDMAI_CMD_BASE_VERSION 0 +#define DPDMAI_CMD_BASE_VERSION 1 #define DPDMAI_CMD_ID_OFFSET 4 -#define DPDMAI_CMDID_FORMAT(x) (((x) << DPDMAI_CMD_ID_OFFSET) | \ - DPDMAI_CMD_BASE_VERSION) +/* + * Maximum number of Tx/Rx queues per DPDMAI object + */ +#define DPDMAI_MAX_QUEUE_NUM 8 + +#define DPDMAI_CMDID_FORMAT_V(x, v) (((x) << DPDMAI_CMD_ID_OFFSET) | (v)) +#define DPDMAI_CMDID_FORMAT(x) DPDMAI_CMDID_FORMAT_V(x, DPDMAI_CMD_BASE_VERSION) /* Command IDs */ #define DPDMAI_CMDID_CLOSE DPDMAI_CMDID_FORMAT(0x800) @@ -26,18 +31,9 @@ #define DPDMAI_CMDID_RESET DPDMAI_CMDID_FORMAT(0x005) #define DPDMAI_CMDID_IS_ENABLED DPDMAI_CMDID_FORMAT(0x006) -#define DPDMAI_CMDID_SET_IRQ DPDMAI_CMDID_FORMAT(0x010) -#define DPDMAI_CMDID_GET_IRQ DPDMAI_CMDID_FORMAT(0x011) -#define DPDMAI_CMDID_SET_IRQ_ENABLE DPDMAI_CMDID_FORMAT(0x012) -#define DPDMAI_CMDID_GET_IRQ_ENABLE DPDMAI_CMDID_FORMAT(0x013) -#define DPDMAI_CMDID_SET_IRQ_MASK DPDMAI_CMDID_FORMAT(0x014) -#define DPDMAI_CMDID_GET_IRQ_MASK DPDMAI_CMDID_FORMAT(0x015) -#define DPDMAI_CMDID_GET_IRQ_STATUS DPDMAI_CMDID_FORMAT(0x016) -#define DPDMAI_CMDID_CLEAR_IRQ_STATUS DPDMAI_CMDID_FORMAT(0x017) - -#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMDID_FORMAT(0x1A0) -#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMDID_FORMAT(0x1A1) -#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMDID_FORMAT(0x1A2) +#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMDID_FORMAT_V(0x1A0, 2) +#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMDID_FORMAT_V(0x1A1, 2) +#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMDID_FORMAT_V(0x1A2, 2) #define MC_CMD_HDR_TOKEN_O 32 /* Token field offset */ #define MC_CMD_HDR_TOKEN_S 16 /* Token field size */ @@ -49,30 +45,32 @@ * Contains initialization APIs and runtime control APIs for DPDMAI */ -/** +/* * Maximum number of Tx/Rx priorities per DPDMAI object */ #define DPDMAI_PRIO_NUM 2 /* DPDMAI queue modification options */ -/** +/* * Select to modify the user's context associated with the queue */ #define DPDMAI_QUEUE_OPT_USER_CTX 0x1 -/** +/* * Select to modify the queue's destination */ #define DPDMAI_QUEUE_OPT_DEST 0x2 /** * struct dpdmai_cfg - Structure representing DPDMAI configuration + * @num_queues: Number of the DMA queues * @priorities: Priorities for the DMA hardware processing; valid priorities are * configured with values 1-8; the entry following last valid entry * should be configured with 0 */ struct dpdmai_cfg { + u8 num_queues; u8 priorities[DPDMAI_PRIO_NUM]; }; @@ -80,20 +78,19 @@ struct dpdmai_cfg { * struct dpdmai_attr - Structure representing DPDMAI attributes * @id: DPDMAI object ID * @version: DPDMAI version + * @version.major: DPDMAI major version + * @version.minor: DPDMAI minor version * @num_of_priorities: number of priorities + * @num_of_queues: number of the DMA queues */ struct dpdmai_attr { int id; - /** - * struct version - DPDMAI version - * @major: DPDMAI major version - * @minor: DPDMAI minor version - */ struct { u16 major; u16 minor; } version; u8 num_of_priorities; + u8 num_of_queues; }; /** @@ -158,22 +155,24 @@ struct dpdmai_rx_queue_attr { u32 fqid; }; +struct dpdmai_tx_queue_attr { + u32 fqid; +}; + int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpdmai_id, u16 *token); int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); -int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); -int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, - const struct dpdmai_cfg *cfg, u16 *token); +int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u32 dpdmai_id, u16 token); int dpdmai_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, struct dpdmai_attr *attr); int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, - u8 priority, const struct dpdmai_rx_queue_cfg *cfg); + u8 queue_idx, u8 priority, const struct dpdmai_rx_queue_cfg *cfg); int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, - u8 priority, struct dpdmai_rx_queue_attr *attr); + u8 queue_idx, u8 priority, struct dpdmai_rx_queue_attr *attr); int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, - u16 token, u8 priority, u32 *fqid); + u16 token, u8 queue_idx, u8 priority, struct dpdmai_tx_queue_attr *attr); #endif /* __FSL_DPDMAI_H */ diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index b18faa7cfedb..3af430787315 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -3,6 +3,8 @@ // Copyright (c) 2013-2014 Freescale Semiconductor, Inc // Copyright (c) 2017 Sysam, Angelo Dureghello <angelo@sysam.it> +#include <linux/cleanup.h> +#include <linux/clk.h> #include <linux/dmapool.h> #include <linux/module.h> #include <linux/slab.h> @@ -74,18 +76,10 @@ static void fsl_edma3_enable_request(struct fsl_edma_chan *fsl_chan) flags = fsl_edma_drvflags(fsl_chan); val = edma_readl_chreg(fsl_chan, ch_sbr); - /* Remote/local swapped wrongly on iMX8 QM Audio edma */ - if (flags & FSL_EDMA_DRV_QUIRK_SWAPPED) { - if (!fsl_chan->is_rxchan) - val |= EDMA_V3_CH_SBR_RD; - else - val |= EDMA_V3_CH_SBR_WR; - } else { - if (fsl_chan->is_rxchan) - val |= EDMA_V3_CH_SBR_RD; - else - val |= EDMA_V3_CH_SBR_WR; - } + if (fsl_chan->is_rxchan) + val |= EDMA_V3_CH_SBR_RD; + else + val |= EDMA_V3_CH_SBR_WR; if (fsl_chan->is_remote) val &= ~(EDMA_V3_CH_SBR_RD | EDMA_V3_CH_SBR_WR); @@ -546,6 +540,8 @@ void fsl_edma_fill_tcd(struct fsl_edma_chan *fsl_chan, csr |= EDMA_TCD_CSR_START; fsl_edma_set_tcd_to_le(fsl_chan, tcd, csr, csr); + + trace_edma_fill_tcd(fsl_chan, tcd); } static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, @@ -810,6 +806,9 @@ int fsl_edma_alloc_chan_resources(struct dma_chan *chan) { struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_CHCLK) + clk_prepare_enable(fsl_chan->clk); + fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_TCD64 ? sizeof(struct fsl_edma_hw_tcd64) : sizeof(struct fsl_edma_hw_tcd), @@ -838,6 +837,8 @@ void fsl_edma_free_chan_resources(struct dma_chan *chan) fsl_chan->tcd_pool = NULL; fsl_chan->is_sw = false; fsl_chan->srcid = 0; + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_CHCLK) + clk_disable_unprepare(fsl_chan->clk); } void fsl_edma_cleanup_vchan(struct dma_device *dmadev) diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h index 7bf0aba471a8..ac66222c1604 100644 --- a/drivers/dma/fsl-edma-common.h +++ b/drivers/dma/fsl-edma-common.h @@ -151,7 +151,6 @@ struct fsl_edma_chan { enum dma_status status; enum fsl_edma_pm_state pm_state; bool idle; - u32 slave_id; struct fsl_edma_engine *edma; struct fsl_edma_desc *edesc; struct dma_slave_config cfg; @@ -195,8 +194,6 @@ struct fsl_edma_desc { #define FSL_EDMA_DRV_HAS_PD BIT(5) #define FSL_EDMA_DRV_HAS_CHCLK BIT(6) #define FSL_EDMA_DRV_HAS_CHMUX BIT(7) -/* imx8 QM audio edma remote local swapped */ -#define FSL_EDMA_DRV_QUIRK_SWAPPED BIT(8) /* control and status register is in tcd address space, edma3 reg layout */ #define FSL_EDMA_DRV_SPLIT_REG BIT(9) #define FSL_EDMA_DRV_BUS_8BYTE BIT(10) @@ -238,7 +235,6 @@ struct fsl_edma_engine { void __iomem *muxbase[DMAMUX_NR]; struct clk *muxclk[DMAMUX_NR]; struct clk *dmaclk; - struct clk *chclk; struct mutex fsl_edma_mutex; const struct fsl_edma_drvdata *drvdata; u32 n_chans; @@ -250,13 +246,17 @@ struct fsl_edma_engine { struct fsl_edma_chan chans[] __counted_by(n_chans); }; +static inline u32 fsl_edma_drvflags(struct fsl_edma_chan *fsl_chan) +{ + return fsl_chan->edma->drvdata->flags; +} + #define edma_read_tcdreg_c(chan, _tcd, __name) \ -(sizeof((_tcd)->__name) == sizeof(u64) ? \ - edma_readq(chan->edma, &(_tcd)->__name) : \ - ((sizeof((_tcd)->__name) == sizeof(u32)) ? \ - edma_readl(chan->edma, &(_tcd)->__name) : \ - edma_readw(chan->edma, &(_tcd)->__name) \ - )) +_Generic(((_tcd)->__name), \ + __iomem __le64 : edma_readq(chan->edma, &(_tcd)->__name), \ + __iomem __le32 : edma_readl(chan->edma, &(_tcd)->__name), \ + __iomem __le16 : edma_readw(chan->edma, &(_tcd)->__name) \ + ) #define edma_read_tcdreg(chan, __name) \ ((fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) ? \ @@ -264,23 +264,13 @@ struct fsl_edma_engine { edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd __iomem *)chan->tcd), __name) \ ) -#define edma_write_tcdreg_c(chan, _tcd, _val, __name) \ -do { \ - switch (sizeof(_tcd->__name)) { \ - case sizeof(u64): \ - edma_writeq(chan->edma, (u64 __force)_val, &_tcd->__name); \ - break; \ - case sizeof(u32): \ - edma_writel(chan->edma, (u32 __force)_val, &_tcd->__name); \ - break; \ - case sizeof(u16): \ - edma_writew(chan->edma, (u16 __force)_val, &_tcd->__name); \ - break; \ - case sizeof(u8): \ - edma_writeb(chan->edma, (u8 __force)_val, &_tcd->__name); \ - break; \ - } \ -} while (0) +#define edma_write_tcdreg_c(chan, _tcd, _val, __name) \ +_Generic((_tcd->__name), \ + __iomem __le64 : edma_writeq(chan->edma, (u64 __force)(_val), &_tcd->__name), \ + __iomem __le32 : edma_writel(chan->edma, (u32 __force)(_val), &_tcd->__name), \ + __iomem __le16 : edma_writew(chan->edma, (u16 __force)(_val), &_tcd->__name), \ + __iomem u8 : edma_writeb(chan->edma, _val, &_tcd->__name) \ + ) #define edma_write_tcdreg(chan, val, __name) \ do { \ @@ -321,9 +311,11 @@ do { \ (((struct fsl_edma_hw_tcd *)_tcd)->_field)) #define fsl_edma_le_to_cpu(x) \ -(sizeof(x) == sizeof(u64) ? le64_to_cpu((__force __le64)(x)) : \ - (sizeof(x) == sizeof(u32) ? le32_to_cpu((__force __le32)(x)) : \ - le16_to_cpu((__force __le16)(x)))) +_Generic((x), \ + __le64 : le64_to_cpu((x)), \ + __le32 : le32_to_cpu((x)), \ + __le16 : le16_to_cpu((x)) \ +) #define fsl_edma_get_tcd_to_cpu(_chan, _tcd, _field) \ (fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ? \ @@ -331,19 +323,11 @@ do { \ fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd *)_tcd)->_field)) #define fsl_edma_set_tcd_to_le_c(_tcd, _val, _field) \ -do { \ - switch (sizeof((_tcd)->_field)) { \ - case sizeof(u64): \ - *(__force __le64 *)(&((_tcd)->_field)) = cpu_to_le64(_val); \ - break; \ - case sizeof(u32): \ - *(__force __le32 *)(&((_tcd)->_field)) = cpu_to_le32(_val); \ - break; \ - case sizeof(u16): \ - *(__force __le16 *)(&((_tcd)->_field)) = cpu_to_le16(_val); \ - break; \ - } \ -} while (0) +_Generic(((_tcd)->_field), \ + __le64 : (_tcd)->_field = cpu_to_le64(_val), \ + __le32 : (_tcd)->_field = cpu_to_le32(_val), \ + __le16 : (_tcd)->_field = cpu_to_le16(_val) \ +) #define fsl_edma_set_tcd_to_le(_chan, _tcd, _val, _field) \ do { \ @@ -353,6 +337,9 @@ do { \ fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd *)_tcd, _val, _field); \ } while (0) +/* Need after struct defination */ +#include "fsl-edma-trace.h" + /* * R/W functions for big- or little-endian registers: * The eDMA controller's endian is independent of the CPU core's endian. @@ -371,23 +358,38 @@ static inline u64 edma_readq(struct fsl_edma_engine *edma, void __iomem *addr) h = ioread32(addr + 4); } + trace_edma_readl(edma, addr, l); + trace_edma_readl(edma, addr + 4, h); + return (h << 32) | l; } static inline u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) { + u32 val; + if (edma->big_endian) - return ioread32be(addr); + val = ioread32be(addr); else - return ioread32(addr); + val = ioread32(addr); + + trace_edma_readl(edma, addr, val); + + return val; } static inline u16 edma_readw(struct fsl_edma_engine *edma, void __iomem *addr) { + u16 val; + if (edma->big_endian) - return ioread16be(addr); + val = ioread16be(addr); else - return ioread16(addr); + val = ioread16(addr); + + trace_edma_readw(edma, addr, val); + + return val; } static inline void edma_writeb(struct fsl_edma_engine *edma, @@ -398,6 +400,8 @@ static inline void edma_writeb(struct fsl_edma_engine *edma, iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); else iowrite8(val, addr); + + trace_edma_writeb(edma, addr, val); } static inline void edma_writew(struct fsl_edma_engine *edma, @@ -408,6 +412,8 @@ static inline void edma_writew(struct fsl_edma_engine *edma, iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); else iowrite16(val, addr); + + trace_edma_writew(edma, addr, val); } static inline void edma_writel(struct fsl_edma_engine *edma, @@ -417,6 +423,8 @@ static inline void edma_writel(struct fsl_edma_engine *edma, iowrite32be(val, addr); else iowrite32(val, addr); + + trace_edma_writel(edma, addr, val); } static inline void edma_writeq(struct fsl_edma_engine *edma, @@ -429,6 +437,9 @@ static inline void edma_writeq(struct fsl_edma_engine *edma, iowrite32(val & 0xFFFFFFFF, addr); iowrite32(val >> 32, addr + 4); } + + trace_edma_writel(edma, addr, val & 0xFFFFFFFF); + trace_edma_writel(edma, addr + 4, val >> 32); } static inline struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) @@ -436,11 +447,6 @@ static inline struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) return container_of(chan, struct fsl_edma_chan, vchan.chan); } -static inline u32 fsl_edma_drvflags(struct fsl_edma_chan *fsl_chan) -{ - return fsl_chan->edma->drvdata->flags; -} - static inline struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) { return container_of(vd, struct fsl_edma_desc, vdesc); diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index 402f0058a180..391e4f13dfeb 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -105,7 +105,8 @@ static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, if (dma_spec->args_count != 2) return NULL; - mutex_lock(&fsl_edma->fsl_edma_mutex); + guard(mutex)(&fsl_edma->fsl_edma_mutex); + list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, device_node) { if (chan->client_count) continue; @@ -114,15 +115,20 @@ static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, if (chan) { chan->device->privatecnt++; fsl_chan = to_fsl_edma_chan(chan); - fsl_chan->slave_id = dma_spec->args[1]; - fsl_edma_chan_mux(fsl_chan, fsl_chan->slave_id, + fsl_chan->srcid = dma_spec->args[1]; + + if (!fsl_chan->srcid) { + dev_err(&fsl_chan->pdev->dev, "Invalidate srcid %d\n", + fsl_chan->srcid); + return NULL; + } + + fsl_edma_chan_mux(fsl_chan, fsl_chan->srcid, true); - mutex_unlock(&fsl_edma->fsl_edma_mutex); return chan; } } } - mutex_unlock(&fsl_edma->fsl_edma_mutex); return NULL; } @@ -342,10 +348,13 @@ static struct fsl_edma_drvdata imx8qm_data = { .setup_irq = fsl_edma3_irq_init, }; -static struct fsl_edma_drvdata imx8qm_audio_data = { - .flags = FSL_EDMA_DRV_QUIRK_SWAPPED | FSL_EDMA_DRV_HAS_PD | FSL_EDMA_DRV_EDMA3, +static struct fsl_edma_drvdata imx8ulp_data = { + .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_CHCLK | FSL_EDMA_DRV_HAS_DMACLK | + FSL_EDMA_DRV_EDMA3, .chreg_space_sz = 0x10000, .chreg_off = 0x10000, + .mux_off = 0x10000 + offsetof(struct fsl_edma3_ch_reg, ch_mux), + .mux_skip = 0x10000, .setup_irq = fsl_edma3_irq_init, }; @@ -380,7 +389,7 @@ static const struct of_device_id fsl_edma_dt_ids[] = { { .compatible = "fsl,ls1028a-edma", .data = &ls1028a_data}, { .compatible = "fsl,imx7ulp-edma", .data = &imx7ulp_data}, { .compatible = "fsl,imx8qm-edma", .data = &imx8qm_data}, - { .compatible = "fsl,imx8qm-adma", .data = &imx8qm_audio_data}, + { .compatible = "fsl,imx8ulp-edma", .data = &imx8ulp_data}, { .compatible = "fsl,imx93-edma3", .data = &imx93_data3}, { .compatible = "fsl,imx93-edma4", .data = &imx93_data4}, { .compatible = "fsl,imx95-edma5", .data = &imx95_data5}, @@ -434,6 +443,7 @@ static int fsl_edma_probe(struct platform_device *pdev) struct fsl_edma_engine *fsl_edma; const struct fsl_edma_drvdata *drvdata = NULL; u32 chan_mask[2] = {0, 0}; + char clk_name[36]; struct edma_regs *regs; int chans; int ret, i; @@ -476,14 +486,6 @@ static int fsl_edma_probe(struct platform_device *pdev) } } - if (drvdata->flags & FSL_EDMA_DRV_HAS_CHCLK) { - fsl_edma->chclk = devm_clk_get_enabled(&pdev->dev, "mp"); - if (IS_ERR(fsl_edma->chclk)) { - dev_err(&pdev->dev, "Missing MP block clock.\n"); - return PTR_ERR(fsl_edma->chclk); - } - } - ret = of_property_read_variable_u32_array(np, "dma-channel-mask", chan_mask, 1, 2); if (ret > 0) { @@ -540,7 +542,7 @@ static int fsl_edma_probe(struct platform_device *pdev) fsl_chan->edma = fsl_edma; fsl_chan->pm_state = RUNNING; - fsl_chan->slave_id = 0; + fsl_chan->srcid = 0; fsl_chan->idle = true; fsl_chan->dma_dir = DMA_NONE; fsl_chan->vchan.desc_free = fsl_edma_free_desc; @@ -551,11 +553,21 @@ static int fsl_edma_probe(struct platform_device *pdev) + i * drvdata->chreg_space_sz + drvdata->chreg_off + len; fsl_chan->mux_addr = fsl_edma->membase + drvdata->mux_off + i * drvdata->mux_skip; + if (drvdata->flags & FSL_EDMA_DRV_HAS_CHCLK) { + snprintf(clk_name, sizeof(clk_name), "ch%02d", i); + fsl_chan->clk = devm_clk_get_enabled(&pdev->dev, + (const char *)clk_name); + + if (IS_ERR(fsl_chan->clk)) + return PTR_ERR(fsl_chan->clk); + } fsl_chan->pdev = pdev; vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev); edma_write_tcdreg(fsl_chan, cpu_to_le32(0), csr); fsl_edma_chan_mux(fsl_chan, 0, false); + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_HAS_CHCLK) + clk_disable_unprepare(fsl_chan->clk); } ret = fsl_edma->drvdata->setup_irq(pdev, fsl_edma); @@ -682,8 +694,8 @@ static int fsl_edma_resume_early(struct device *dev) continue; fsl_chan->pm_state = RUNNING; edma_write_tcdreg(fsl_chan, 0, csr); - if (fsl_chan->slave_id != 0) - fsl_edma_chan_mux(fsl_chan, fsl_chan->slave_id, true); + if (fsl_chan->srcid != 0) + fsl_edma_chan_mux(fsl_chan, fsl_chan->srcid, true); } if (!(fsl_edma->drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) diff --git a/drivers/dma/fsl-edma-trace.c b/drivers/dma/fsl-edma-trace.c new file mode 100644 index 000000000000..28300ad80bb7 --- /dev/null +++ b/drivers/dma/fsl-edma-trace.c @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define CREATE_TRACE_POINTS +#include "fsl-edma-common.h" diff --git a/drivers/dma/fsl-edma-trace.h b/drivers/dma/fsl-edma-trace.h new file mode 100644 index 000000000000..d3541301a247 --- /dev/null +++ b/drivers/dma/fsl-edma-trace.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright 2023 NXP. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM fsl_edma + +#if !defined(__LINUX_FSL_EDMA_TRACE) || defined(TRACE_HEADER_MULTI_READ) +#define __LINUX_FSL_EDMA_TRACE + +#include <linux/types.h> +#include <linux/tracepoint.h> + +DECLARE_EVENT_CLASS(edma_log_io, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value), + TP_STRUCT__entry( + __field(struct fsl_edma_engine *, edma) + __field(void __iomem *, addr) + __field(u32, value) + ), + TP_fast_assign( + __entry->edma = edma; + __entry->addr = addr; + __entry->value = value; + ), + TP_printk("offset %08x: value %08x", + (u32)(__entry->addr - __entry->edma->membase), __entry->value) +); + +DEFINE_EVENT(edma_log_io, edma_readl, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_writel, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_readw, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_writew, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_readb, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_writeb, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DECLARE_EVENT_CLASS(edma_log_tcd, + TP_PROTO(struct fsl_edma_chan *chan, void *tcd), + TP_ARGS(chan, tcd), + TP_STRUCT__entry( + __field(u64, saddr) + __field(u16, soff) + __field(u16, attr) + __field(u32, nbytes) + __field(u64, slast) + __field(u64, daddr) + __field(u16, doff) + __field(u16, citer) + __field(u64, dlast_sga) + __field(u16, csr) + __field(u16, biter) + + ), + TP_fast_assign( + __entry->saddr = fsl_edma_get_tcd_to_cpu(chan, tcd, saddr), + __entry->soff = fsl_edma_get_tcd_to_cpu(chan, tcd, soff), + __entry->attr = fsl_edma_get_tcd_to_cpu(chan, tcd, attr), + __entry->nbytes = fsl_edma_get_tcd_to_cpu(chan, tcd, nbytes), + __entry->slast = fsl_edma_get_tcd_to_cpu(chan, tcd, slast), + __entry->daddr = fsl_edma_get_tcd_to_cpu(chan, tcd, daddr), + __entry->doff = fsl_edma_get_tcd_to_cpu(chan, tcd, doff), + __entry->citer = fsl_edma_get_tcd_to_cpu(chan, tcd, citer), + __entry->dlast_sga = fsl_edma_get_tcd_to_cpu(chan, tcd, dlast_sga), + __entry->csr = fsl_edma_get_tcd_to_cpu(chan, tcd, csr), + __entry->biter = fsl_edma_get_tcd_to_cpu(chan, tcd, biter); + ), + TP_printk("\n==== TCD =====\n" + " saddr: 0x%016llx\n" + " soff: 0x%04x\n" + " attr: 0x%04x\n" + " nbytes: 0x%08x\n" + " slast: 0x%016llx\n" + " daddr: 0x%016llx\n" + " doff: 0x%04x\n" + " citer: 0x%04x\n" + " dlast: 0x%016llx\n" + " csr: 0x%04x\n" + " biter: 0x%04x\n", + __entry->saddr, + __entry->soff, + __entry->attr, + __entry->nbytes, + __entry->slast, + __entry->daddr, + __entry->doff, + __entry->citer, + __entry->dlast_sga, + __entry->csr, + __entry->biter) +); + +DEFINE_EVENT(edma_log_tcd, edma_fill_tcd, + TP_PROTO(struct fsl_edma_chan *chan, void *tcd), + TP_ARGS(chan, tcd) +); + +#endif + +/* this part must be outside header guard */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE fsl-edma-trace + +#include <trace/define_trace.h> diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c index 1398814d8fbb..e3505e56784b 100644 --- a/drivers/dma/idma64.c +++ b/drivers/dma/idma64.c @@ -598,7 +598,9 @@ static int idma64_probe(struct idma64_chip *chip) idma64->dma.dev = chip->sysdev; - dma_set_max_seg_size(idma64->dma.dev, IDMA64C_CTLH_BLOCK_TS_MASK); + ret = dma_set_max_seg_size(idma64->dma.dev, IDMA64C_CTLH_BLOCK_TS_MASK); + if (ret) + return ret; ret = dma_async_device_register(&idma64->dma); if (ret) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 39935071174a..57f1bf2ab20b 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -577,7 +577,6 @@ void idxd_wq_del_cdev(struct idxd_wq *wq) struct idxd_cdev *idxd_cdev; idxd_cdev = wq->idxd_cdev; - ida_destroy(&file_ida); wq->idxd_cdev = NULL; cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); put_device(cdev_dev(idxd_cdev)); @@ -593,6 +592,14 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) if (idxd->state != IDXD_DEV_ENABLED) return -ENXIO; + mutex_lock(&wq->wq_lock); + + if (!idxd_wq_driver_name_match(wq, dev)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; + rc = -ENODEV; + goto wq_err; + } + /* * User type WQ is enabled only when SVA is enabled for two reasons: * - If no IOMMU or IOMMU Passthrough without SVA, userspace @@ -608,14 +615,7 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) dev_dbg(&idxd->pdev->dev, "User type WQ cannot be enabled without SVA.\n"); - return -EOPNOTSUPP; - } - - mutex_lock(&wq->wq_lock); - - if (!idxd_wq_driver_name_match(wq, dev)) { - idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; - rc = -ENODEV; + rc = -EOPNOTSUPP; goto wq_err; } diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 9b42f5e96b1e..003e1580b902 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -24,6 +24,7 @@ #include <linux/semaphore.h> #include <linux/spinlock.h> #include <linux/device.h> +#include <linux/genalloc.h> #include <linux/dma-mapping.h> #include <linux/firmware.h> #include <linux/slab.h> @@ -137,7 +138,11 @@ * 0: Source on AIPS * 12 Destination Bit(DP) 1: Destination on SPBA * 0: Destination on AIPS - * 13-15 --------- MUST BE 0 + * 13 Source FIFO 1: Source is dual FIFO + * 0: Source is single FIFO + * 14 Destination FIFO 1: Destination is dual FIFO + * 0: Destination is single FIFO + * 15 --------- MUST BE 0 * 16-23 Higher WML HWML * 24-27 N Total number of samples after * which Pad adding/Swallowing @@ -168,6 +173,8 @@ #define SDMA_WATERMARK_LEVEL_SPDIF BIT(10) #define SDMA_WATERMARK_LEVEL_SP BIT(11) #define SDMA_WATERMARK_LEVEL_DP BIT(12) +#define SDMA_WATERMARK_LEVEL_SD BIT(13) +#define SDMA_WATERMARK_LEVEL_DD BIT(14) #define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16) #define SDMA_WATERMARK_LEVEL_LWE BIT(28) #define SDMA_WATERMARK_LEVEL_HWE BIT(29) @@ -175,6 +182,7 @@ #define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) #define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \ @@ -232,20 +240,23 @@ struct sdma_script_start_addrs { s32 utra_addr; s32 ram_code_start_addr; /* End of v1 array */ - s32 mcu_2_ssish_addr; + union { s32 v1_end; s32 mcu_2_ssish_addr; }; s32 ssish_2_mcu_addr; s32 hdmi_dma_addr; /* End of v2 array */ - s32 zcanfd_2_mcu_addr; + union { s32 v2_end; s32 zcanfd_2_mcu_addr; }; s32 zqspi_2_mcu_addr; s32 mcu_2_ecspi_addr; s32 mcu_2_sai_addr; s32 sai_2_mcu_addr; s32 uart_2_mcu_rom_addr; s32 uartsh_2_mcu_rom_addr; + s32 i2c_2_mcu_addr; + s32 mcu_2_i2c_addr; /* End of v3 array */ - s32 mcu_2_zqspi_addr; + union { s32 v3_end; s32 mcu_2_zqspi_addr; }; /* End of v4 array */ + s32 v4_end[0]; }; /* @@ -531,6 +542,7 @@ struct sdma_engine { /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/ bool clk_ratio; bool fw_loaded; + struct gen_pool *iram_pool; }; static int sdma_config_write(struct dma_chan *chan, @@ -1072,6 +1084,11 @@ static int sdma_get_pc(struct sdma_channel *sdmac, per_2_emi = sdma->script_addrs->sai_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_sai_addr; break; + case IMX_DMATYPE_I2C: + per_2_emi = sdma->script_addrs->i2c_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_i2c_addr; + sdmac->is_ram_script = true; + break; case IMX_DMATYPE_HDMI: emi_2_per = sdma->script_addrs->hdmi_dma_addr; sdmac->is_ram_script = true; @@ -1255,6 +1272,16 @@ static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP; sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT; + + /* + * Limitation: The p2p script support dual fifos in maximum, + * So when fifo number is larger than 1, force enable dual + * fifos. + */ + if (sdmac->n_fifos_src > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SD; + if (sdmac->n_fifos_dst > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD; } static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac) @@ -1358,8 +1385,14 @@ static int sdma_request_channel0(struct sdma_engine *sdma) { int ret = -EBUSY; - sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys, - GFP_NOWAIT); + if (sdma->iram_pool) + sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys); + else + sdma->bd0 = dma_alloc_coherent(sdma->dev, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys, GFP_NOWAIT); if (!sdma->bd0) { ret = -ENOMEM; goto out; @@ -1379,10 +1412,14 @@ out: static int sdma_alloc_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; int ret = 0; - desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size, - &desc->bd_phys, GFP_NOWAIT); + if (sdma->iram_pool) + desc->bd = gen_pool_dma_alloc(sdma->iram_pool, bd_size, &desc->bd_phys); + else + desc->bd = dma_alloc_coherent(sdma->dev, bd_size, &desc->bd_phys, GFP_NOWAIT); + if (!desc->bd) { ret = -ENOMEM; goto out; @@ -1394,9 +1431,12 @@ out: static void sdma_free_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; - dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, - desc->bd_phys); + if (sdma->iram_pool) + gen_pool_free(sdma->iram_pool, (unsigned long)desc->bd, bd_size); + else + dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, desc->bd_phys); } static void sdma_desc_free(struct virt_dma_desc *vd) @@ -1643,6 +1683,9 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( if (count & 3 || sg->dma_address & 3) goto err_bd_out; break; + case DMA_SLAVE_BUSWIDTH_3_BYTES: + bd->mode.command = 3; + break; case DMA_SLAVE_BUSWIDTH_2_BYTES: bd->mode.command = 2; if (count & 1 || sg->dma_address & 1) @@ -1880,10 +1923,17 @@ static void sdma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&sdmac->vc.lock, flags); } -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 45 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 46 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 \ +(offsetof(struct sdma_script_start_addrs, v1_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 \ +(offsetof(struct sdma_script_start_addrs, v2_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 \ +(offsetof(struct sdma_script_start_addrs, v3_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 \ +(offsetof(struct sdma_script_start_addrs, v4_end) / sizeof(s32)) static void sdma_add_scripts(struct sdma_engine *sdma, const struct sdma_script_start_addrs *addr) @@ -2068,6 +2118,7 @@ static int sdma_init(struct sdma_engine *sdma) { int i, ret; dma_addr_t ccb_phys; + int ccbsize; ret = clk_enable(sdma->clk_ipg); if (ret) @@ -2083,10 +2134,14 @@ static int sdma_init(struct sdma_engine *sdma) /* Be sure SDMA has not started yet */ writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); - sdma->channel_control = dma_alloc_coherent(sdma->dev, - MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control) + - sizeof(struct sdma_context_data), - &ccb_phys, GFP_KERNEL); + ccbsize = MAX_DMA_CHANNELS * (sizeof(struct sdma_channel_control) + + sizeof(struct sdma_context_data)); + + if (sdma->iram_pool) + sdma->channel_control = gen_pool_dma_alloc(sdma->iram_pool, ccbsize, &ccb_phys); + else + sdma->channel_control = dma_alloc_coherent(sdma->dev, ccbsize, &ccb_phys, + GFP_KERNEL); if (!sdma->channel_control) { ret = -ENOMEM; @@ -2272,6 +2327,12 @@ static int sdma_probe(struct platform_device *pdev) vchan_init(&sdmac->vc, &sdma->dma_device); } + if (np) { + sdma->iram_pool = of_gen_pool_get(np, "iram", 0); + if (sdma->iram_pool) + dev_info(&pdev->dev, "alloc bd from iram.\n"); + } + ret = sdma_init(sdma); if (ret) goto err_init; diff --git a/drivers/dma/mcf-edma-main.c b/drivers/dma/mcf-edma-main.c index dba631783876..78c606f6d002 100644 --- a/drivers/dma/mcf-edma-main.c +++ b/drivers/dma/mcf-edma-main.c @@ -195,7 +195,7 @@ static int mcf_edma_probe(struct platform_device *pdev) struct fsl_edma_chan *mcf_chan = &mcf_edma->chans[i]; mcf_chan->edma = mcf_edma; - mcf_chan->slave_id = i; + mcf_chan->srcid = i; mcf_chan->idle = true; mcf_chan->dma_dir = DMA_NONE; mcf_chan->vchan.desc_free = fsl_edma_free_desc; @@ -277,7 +277,7 @@ bool mcf_edma_filter_fn(struct dma_chan *chan, void *param) if (chan->device->dev->driver == &mcf_edma_driver.driver) { struct fsl_edma_chan *mcf_chan = to_fsl_edma_chan(chan); - return (mcf_chan->slave_id == (uintptr_t)param); + return (mcf_chan->srcid == (uintptr_t)param); } return false; diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index c359decc07a3..6b2793b07694 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -155,11 +155,6 @@ static inline struct device *chan2dev(struct dma_chan *chan) return &chan->dev->device; } -static inline struct device *chan2parent(struct dma_chan *chan) -{ - return chan->dev->device.parent; -} - static inline struct pch_dma_desc *pdc_first_active(struct pch_dma_chan *pd_chan) { diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index ad8e3da1b2cd..60c4de8dac1d 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -3262,7 +3262,6 @@ MODULE_DEVICE_TABLE(amba, pl330_ids); static struct amba_driver pl330_driver = { .drv = { - .owner = THIS_MODULE, .name = "dma-pl330", .pm = &pl330_pm, }, diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index 202ac95227cb..721b4ac0857a 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -50,7 +50,6 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> -#include <linux/of_dma.h> #include <linux/property.h> #include <linux/delay.h> #include <linux/acpi.h> @@ -947,22 +946,12 @@ static const struct acpi_device_id hidma_acpi_ids[] = { MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids); #endif -static const struct of_device_id hidma_match[] = { - {.compatible = "qcom,hidma-1.0",}, - {.compatible = "qcom,hidma-1.1", .data = (void *)(HIDMA_MSI_CAP),}, - {.compatible = "qcom,hidma-1.2", - .data = (void *)(HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP),}, - {}, -}; -MODULE_DEVICE_TABLE(of, hidma_match); - static struct platform_driver hidma_driver = { .probe = hidma_probe, .remove_new = hidma_remove, .shutdown = hidma_shutdown, .driver = { .name = "hidma", - .of_match_table = hidma_match, .acpi_match_table = ACPI_PTR(hidma_acpi_ids), }, }; diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c index 1d675f31252b..bb883e138ebf 100644 --- a/drivers/dma/qcom/hidma_mgmt.c +++ b/drivers/dma/qcom/hidma_mgmt.c @@ -7,12 +7,7 @@ #include <linux/dmaengine.h> #include <linux/acpi.h> -#include <linux/of.h> #include <linux/property.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/of_platform.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/module.h> #include <linux/uaccess.h> @@ -327,115 +322,13 @@ static const struct acpi_device_id hidma_mgmt_acpi_ids[] = { MODULE_DEVICE_TABLE(acpi, hidma_mgmt_acpi_ids); #endif -static const struct of_device_id hidma_mgmt_match[] = { - {.compatible = "qcom,hidma-mgmt-1.0",}, - {}, -}; -MODULE_DEVICE_TABLE(of, hidma_mgmt_match); - static struct platform_driver hidma_mgmt_driver = { .probe = hidma_mgmt_probe, .driver = { .name = "hidma-mgmt", - .of_match_table = hidma_mgmt_match, .acpi_match_table = ACPI_PTR(hidma_mgmt_acpi_ids), }, }; -#if defined(CONFIG_OF) && defined(CONFIG_OF_IRQ) -static int object_counter; - -static int __init hidma_mgmt_of_populate_channels(struct device_node *np) -{ - struct platform_device *pdev_parent = of_find_device_by_node(np); - struct platform_device_info pdevinfo; - struct device_node *child; - struct resource *res; - int ret = 0; - - /* allocate a resource array */ - res = kcalloc(3, sizeof(*res), GFP_KERNEL); - if (!res) - return -ENOMEM; - - for_each_available_child_of_node(np, child) { - struct platform_device *new_pdev; - - ret = of_address_to_resource(child, 0, &res[0]); - if (!ret) - goto out; - - ret = of_address_to_resource(child, 1, &res[1]); - if (!ret) - goto out; - - ret = of_irq_to_resource(child, 0, &res[2]); - if (ret <= 0) - goto out; - - memset(&pdevinfo, 0, sizeof(pdevinfo)); - pdevinfo.fwnode = &child->fwnode; - pdevinfo.parent = pdev_parent ? &pdev_parent->dev : NULL; - pdevinfo.name = child->name; - pdevinfo.id = object_counter++; - pdevinfo.res = res; - pdevinfo.num_res = 3; - pdevinfo.data = NULL; - pdevinfo.size_data = 0; - pdevinfo.dma_mask = DMA_BIT_MASK(64); - new_pdev = platform_device_register_full(&pdevinfo); - if (IS_ERR(new_pdev)) { - ret = PTR_ERR(new_pdev); - goto out; - } - new_pdev->dev.of_node = child; - of_dma_configure(&new_pdev->dev, child, true); - /* - * It is assumed that calling of_msi_configure is safe on - * platforms with or without MSI support. - */ - of_msi_configure(&new_pdev->dev, child); - } - - kfree(res); - - return ret; - -out: - of_node_put(child); - kfree(res); - - return ret; -} -#endif - -static int __init hidma_mgmt_init(void) -{ -#if defined(CONFIG_OF) && defined(CONFIG_OF_IRQ) - struct device_node *child; - - for_each_matching_node(child, hidma_mgmt_match) { - /* device tree based firmware here */ - hidma_mgmt_of_populate_channels(child); - } -#endif - /* - * We do not check for return value here, as it is assumed that - * platform_driver_register must not fail. The reason for this is that - * the (potential) hidma_mgmt_of_populate_channels calls above are not - * cleaned up if it does fail, and to do this work is quite - * complicated. In particular, various calls of of_address_to_resource, - * of_irq_to_resource, platform_device_register_full, of_dma_configure, - * and of_msi_configure which then call other functions and so on, must - * be cleaned up - this is not a trivial exercise. - * - * Currently, this module is not intended to be unloaded, and there is - * no module_exit function defined which does the needed cleanup. For - * this reason, we have to assume success here. - */ - platform_driver_register(&hidma_mgmt_driver); - - return 0; -} -module_init(hidma_mgmt_init); +module_platform_driver(hidma_mgmt_driver); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h index e9f5250fbe4d..59d9eabc8b67 100644 --- a/drivers/dma/virt-dma.h +++ b/drivers/dma/virt-dma.h @@ -81,6 +81,8 @@ static inline struct dma_async_tx_descriptor *vchan_tx_prep(struct virt_dma_chan */ static inline bool vchan_issue_pending(struct virt_dma_chan *vc) { + lockdep_assert_held(&vc->lock); + list_splice_tail_init(&vc->desc_submitted, &vc->desc_issued); return !list_empty(&vc->desc_issued); } @@ -96,6 +98,8 @@ static inline void vchan_cookie_complete(struct virt_dma_desc *vd) struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); dma_cookie_t cookie; + lockdep_assert_held(&vc->lock); + cookie = vd->tx.cookie; dma_cookie_complete(&vd->tx); dev_vdbg(vc->chan.device->dev, "txd %p[%x]: marked complete\n", @@ -146,6 +150,8 @@ static inline void vchan_terminate_vdesc(struct virt_dma_desc *vd) { struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); + lockdep_assert_held(&vc->lock); + list_add_tail(&vd->node, &vc->desc_terminated); if (vc->cyclic == vd) @@ -160,6 +166,8 @@ static inline void vchan_terminate_vdesc(struct virt_dma_desc *vd) */ static inline struct virt_dma_desc *vchan_next_desc(struct virt_dma_chan *vc) { + lockdep_assert_held(&vc->lock); + return list_first_entry_or_null(&vc->desc_issued, struct virt_dma_desc, node); } @@ -177,6 +185,8 @@ static inline struct virt_dma_desc *vchan_next_desc(struct virt_dma_chan *vc) static inline void vchan_get_all_descriptors(struct virt_dma_chan *vc, struct list_head *head) { + lockdep_assert_held(&vc->lock); + list_splice_tail_init(&vc->desc_allocated, head); list_splice_tail_init(&vc->desc_submitted, head); list_splice_tail_init(&vc->desc_issued, head); diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 313b217388fe..e143a7330816 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -1307,6 +1307,7 @@ static const struct platform_device_id xdma_id_table[] = { { "xdma", 0}, { }, }; +MODULE_DEVICE_TABLE(platform, xdma_id_table); static struct platform_driver xdma_driver = { .driver = { diff --git a/drivers/dma/xilinx/xilinx_dpdma.c b/drivers/dma/xilinx/xilinx_dpdma.c index eb0637d90342..36bd4825d389 100644 --- a/drivers/dma/xilinx/xilinx_dpdma.c +++ b/drivers/dma/xilinx/xilinx_dpdma.c @@ -1043,9 +1043,8 @@ static int xilinx_dpdma_chan_stop(struct xilinx_dpdma_chan *chan) static void xilinx_dpdma_chan_done_irq(struct xilinx_dpdma_chan *chan) { struct xilinx_dpdma_tx_desc *active; - unsigned long flags; - spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->lock); xilinx_dpdma_debugfs_desc_done_irq(chan); @@ -1057,7 +1056,7 @@ static void xilinx_dpdma_chan_done_irq(struct xilinx_dpdma_chan *chan) "chan%u: DONE IRQ with no active descriptor!\n", chan->id); - spin_unlock_irqrestore(&chan->lock, flags); + spin_unlock(&chan->lock); } /** @@ -1072,10 +1071,9 @@ static void xilinx_dpdma_chan_vsync_irq(struct xilinx_dpdma_chan *chan) { struct xilinx_dpdma_tx_desc *pending; struct xilinx_dpdma_sw_desc *sw_desc; - unsigned long flags; u32 desc_id; - spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->lock); pending = chan->desc.pending; if (!chan->running || !pending) @@ -1108,7 +1106,7 @@ static void xilinx_dpdma_chan_vsync_irq(struct xilinx_dpdma_chan *chan) spin_unlock(&chan->vchan.lock); out: - spin_unlock_irqrestore(&chan->lock, flags); + spin_unlock(&chan->lock); } /** diff --git a/drivers/firmware/dmi-id.c b/drivers/firmware/dmi-id.c index 5f3a3e913d28..d19c78a78ae3 100644 --- a/drivers/firmware/dmi-id.c +++ b/drivers/firmware/dmi-id.c @@ -169,9 +169,14 @@ static int dmi_dev_uevent(const struct device *dev, struct kobj_uevent_env *env) return 0; } +static void dmi_dev_release(struct device *dev) +{ + kfree(dev); +} + static struct class dmi_class = { .name = "dmi", - .dev_release = (void(*)(struct device *)) kfree, + .dev_release = dmi_dev_release, .dev_uevent = dmi_dev_uevent, }; diff --git a/drivers/firmware/dmi_scan.c b/drivers/firmware/dmi_scan.c index 015c95a825d3..68231c638c55 100644 --- a/drivers/firmware/dmi_scan.c +++ b/drivers/firmware/dmi_scan.c @@ -42,6 +42,7 @@ static struct dmi_memdev_info { u8 type; /* DDR2, DDR3, DDR4 etc */ } *dmi_memdev; static int dmi_memdev_nr; +static int dmi_memdev_populated_nr __initdata; static const char * __init dmi_string_nosave(const struct dmi_header *dm, u8 s) { @@ -102,6 +103,17 @@ static void dmi_decode_table(u8 *buf, const struct dmi_header *dm = (const struct dmi_header *)data; /* + * If a short entry is found (less than 4 bytes), not only it + * is invalid, but we cannot reliably locate the next entry. + */ + if (dm->length < sizeof(struct dmi_header)) { + pr_warn(FW_BUG + "Corrupted DMI table, offset %zd (only %d entries processed)\n", + data - buf, i); + break; + } + + /* * We want to know the total length (formatted area and * strings) before decoding to make sure we won't run off the * table in dmi_decode or dmi_string @@ -448,6 +460,9 @@ static void __init save_mem_devices(const struct dmi_header *dm, void *v) else bytes = (u64)get_unaligned((u32 *)&d[0x1C]) << 20; + if (bytes) + dmi_memdev_populated_nr++; + dmi_memdev[nr].size = bytes; nr++; } @@ -824,6 +839,8 @@ void __init dmi_setup(void) return; dmi_memdev_walk(); + pr_info("DMI: Memory slots populated: %d/%d\n", + dmi_memdev_populated_nr, dmi_memdev_nr); dump_stack_set_arch_desc("%s", dmi_ids_string); } diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 31eb1e287ce1..06f0428a723c 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -56,17 +56,6 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_CFI), $(KBUILD_CFLAGS)) # disable LTO KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS)) -GCOV_PROFILE := n -# Sanitizer runtimes are unavailable and cannot be linked here. -KASAN_SANITIZE := n -KCSAN_SANITIZE := n -KMSAN_SANITIZE := n -UBSAN_SANITIZE := n -OBJECT_FILES_NON_STANDARD := y - -# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. -KCOV_INSTRUMENT := n - lib-y := efi-stub-helper.o gop.o secureboot.o tpm.o \ file.o mem.o random.o randomalloc.o pci.o \ skip_spaces.o lib-cmdline.o lib-ctype.o \ diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 553a5f94c00a..bb063b81cee6 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -128,7 +128,24 @@ static bool acpi_gpio_deferred_req_irqs_done; static int acpi_gpiochip_find(struct gpio_chip *gc, const void *data) { - return device_match_acpi_handle(&gc->gpiodev->dev, data); + /* First check the actual GPIO device */ + if (device_match_acpi_handle(&gc->gpiodev->dev, data)) + return true; + + /* + * When the ACPI device is artificially split to the banks of GPIOs, + * where each of them is represented by a separate GPIO device, + * the firmware node of the physical device may not be shared among + * the banks as they may require different values for the same property, + * e.g., number of GPIOs in a certain bank. In such case the ACPI handle + * of a GPIO device is NULL and can not be used. Hence we have to check + * the parent device to be sure that there is no match before bailing + * out. + */ + if (gc->parent) + return device_match_acpi_handle(gc->parent, data); + + return false; } /** @@ -938,6 +955,10 @@ static struct gpio_desc *acpi_get_gpiod_from_data(struct fwnode_handle *fwnode, static bool acpi_can_fallback_to_crs(struct acpi_device *adev, const char *con_id) { + /* If there is no ACPI device, there is no _CRS to fall back to */ + if (!adev) + return false; + /* Never allow fallback if the device has properties */ if (acpi_dev_has_props(adev) || adev->driver_gpios) return false; @@ -978,10 +999,10 @@ __acpi_find_gpio(struct fwnode_handle *fwnode, const char *con_id, unsigned int } /* Then from plain _CRS GPIOs */ - if (!adev || !can_fallback) - return ERR_PTR(-ENOENT); + if (can_fallback) + return acpi_get_gpiod_by_index(adev, NULL, idx, info); - return acpi_get_gpiod_by_index(adev, NULL, idx, info); + return ERR_PTR(-ENOENT); } struct gpio_desc *acpi_find_gpio(struct fwnode_handle *fwnode, diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 1f6b56ec99f6..078f588e99eb 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -23,7 +23,7 @@ # Makefile for the drm device driver. This driver provides support for the # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. -FULL_AMD_PATH=$(srctree)/$(src)/.. +FULL_AMD_PATH=$(src)/.. DISPLAY_FOLDER_NAME=display FULL_AMD_DISPLAY_PATH = $(FULL_AMD_PATH)/$(DISPLAY_FOLDER_NAME) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 665c63f55278..013ff373e067 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -279,7 +279,7 @@ int amdgpu_irq_init(struct amdgpu_device *adev) adev->irq.msi_enabled = false; if (!amdgpu_msi_ok(adev)) - flags = PCI_IRQ_LEGACY; + flags = PCI_IRQ_INTX; else flags = PCI_IRQ_ALL_TYPES; diff --git a/drivers/gpu/drm/arm/display/komeda/Makefile b/drivers/gpu/drm/arm/display/komeda/Makefile index 1931a7fa1a14..cf5287fcbbc2 100644 --- a/drivers/gpu/drm/arm/display/komeda/Makefile +++ b/drivers/gpu/drm/arm/display/komeda/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 ccflags-y := \ - -I $(srctree)/$(src)/../include \ - -I $(srctree)/$(src) + -I $(src)/../include \ + -I $(src) komeda-y := \ komeda_drv.o \ diff --git a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c index 8f84e98249c7..2fbeda9025bf 100644 --- a/drivers/gpu/drm/bridge/sii902x.c +++ b/drivers/gpu/drm/bridge/sii902x.c @@ -1092,7 +1092,7 @@ static int sii902x_init(struct sii902x *sii902x) } sii902x->i2cmux->priv = sii902x; - ret = i2c_mux_add_adapter(sii902x->i2cmux, 0, 0, 0); + ret = i2c_mux_add_adapter(sii902x->i2cmux, 0, 0); if (ret) goto err_unreg_audio; diff --git a/drivers/gpu/drm/gma500/mmu.c b/drivers/gpu/drm/gma500/mmu.c index a70b01ccdf70..4d78b33eaa82 100644 --- a/drivers/gpu/drm/gma500/mmu.c +++ b/drivers/gpu/drm/gma500/mmu.c @@ -5,6 +5,7 @@ **************************************************************************/ #include <linux/highmem.h> +#include <linux/vmalloc.h> #include "mmu.h" #include "psb_drv.h" diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 7cad944b825c..c8c8b31da4fb 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -36,7 +36,7 @@ subdir-ccflags-$(CONFIG_DRM_I915_WERROR) += -Werror # drivers. Define I915 when building i915. subdir-ccflags-y += -DI915 -subdir-ccflags-y += -I$(srctree)/$(src) +subdir-ccflags-y += -I$(src) # Please keep these build lists sorted! @@ -430,7 +430,7 @@ no-header-test := \ always-$(CONFIG_DRM_I915_WERROR) += \ $(patsubst %.h,%.hdrtest, $(filter-out $(no-header-test), \ - $(shell cd $(srctree)/$(src) && find * -name '*.h'))) + $(shell cd $(src) && find * -name '*.h'))) quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) cmd_hdrtest = $(CC) $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; \ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index 0ba955611dfb..8780aa243105 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -5,6 +5,7 @@ */ #include <drm/drm_cache.h> +#include <linux/vmalloc.h> #include "gt/intel_gt.h" #include "gt/intel_tlb.h" diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c b/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c index b2a5882b8f81..075657018739 100644 --- a/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c @@ -4,6 +4,7 @@ * Copyright © 2016 Intel Corporation */ +#include <linux/vmalloc.h> #include "mock_dmabuf.h" static struct sg_table *mock_map_dma_buf(struct dma_buf_attachment *attachment, diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c index bccc3a1200bc..1fb6ff77fd89 100644 --- a/drivers/gpu/drm/i915/gt/shmem_utils.c +++ b/drivers/gpu/drm/i915/gt/shmem_utils.c @@ -7,6 +7,7 @@ #include <linux/mm.h> #include <linux/pagemap.h> #include <linux/shmem_fs.h> +#include <linux/vmalloc.h> #include "i915_drv.h" #include "gem/i915_gem_object.h" diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c index 4dd52ac2043e..d800d267f0e9 100644 --- a/drivers/gpu/drm/i915/gvt/firmware.c +++ b/drivers/gpu/drm/i915/gvt/firmware.c @@ -30,6 +30,7 @@ #include <linux/firmware.h> #include <linux/crc32.h> +#include <linux/vmalloc.h> #include "i915_drv.h" #include "gvt.h" diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index 094fca9b0e73..58cca4906f41 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -39,6 +39,7 @@ #include "trace.h" #include "gt/intel_gt_regs.h" +#include <linux/vmalloc.h> #if defined(VERBOSE_DEBUG) #define gvt_vdbg_mm(fmt, args...) gvt_dbg_mm(fmt, ##args) diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index 102eb354fed6..22fbddbe3e23 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -54,6 +54,7 @@ #include "display/skl_watermark_regs.h" #include "display/vlv_dsi_pll_regs.h" #include "gt/intel_gt_regs.h" +#include <linux/vmalloc.h> /* XXX FIXME i915 has changed PP_XXX definition */ #define PCH_PP_STATUS _MMIO(0xc7200) diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c index 922711e0e30b..e16e0d4c9534 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.c +++ b/drivers/gpu/drm/i915/gvt/mmio.c @@ -33,6 +33,7 @@ * */ +#include <linux/vmalloc.h> #include "i915_drv.h" #include "i915_reg.h" #include "gvt.h" diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index 08ad1bd651f1..63c751ca4119 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -34,6 +34,7 @@ #include "i915_drv.h" #include "gvt.h" #include "i915_pvinfo.h" +#include <linux/vmalloc.h> void populate_pvinfo_page(struct intel_vgpu *vgpu) { diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c index 9b6d87c8b583..5a01d60e5186 100644 --- a/drivers/gpu/drm/i915/intel_gvt.c +++ b/drivers/gpu/drm/i915/intel_gvt.c @@ -28,6 +28,7 @@ #include "gt/intel_context.h" #include "gt/intel_ring.h" #include "gt/shmem_utils.h" +#include <linux/vmalloc.h> /** * DOC: Intel GVT-g host support diff --git a/drivers/gpu/drm/imagination/Makefile b/drivers/gpu/drm/imagination/Makefile index ec6db8e9b403..9bc6a3884c22 100644 --- a/drivers/gpu/drm/imagination/Makefile +++ b/drivers/gpu/drm/imagination/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT # Copyright (c) 2023 Imagination Technologies Ltd. -subdir-ccflags-y := -I$(srctree)/$(src) +subdir-ccflags-y := -I$(src) powervr-y := \ pvr_ccb.o \ diff --git a/drivers/gpu/drm/imagination/pvr_vm_mips.c b/drivers/gpu/drm/imagination/pvr_vm_mips.c index 4f99b4af871c..94af854547d6 100644 --- a/drivers/gpu/drm/imagination/pvr_vm_mips.c +++ b/drivers/gpu/drm/imagination/pvr_vm_mips.c @@ -14,6 +14,7 @@ #include <linux/err.h> #include <linux/slab.h> #include <linux/types.h> +#include <linux/vmalloc.h> /** * pvr_vm_mips_init() - Initialise MIPS FW pagetable diff --git a/drivers/gpu/drm/mediatek/mtk_gem.c b/drivers/gpu/drm/mediatek/mtk_gem.c index 5a82d7cf3ed0..a172456d1d7b 100644 --- a/drivers/gpu/drm/mediatek/mtk_gem.c +++ b/drivers/gpu/drm/mediatek/mtk_gem.c @@ -4,6 +4,7 @@ */ #include <linux/dma-buf.h> +#include <linux/vmalloc.h> #include <drm/drm.h> #include <drm/drm_device.h> diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile index 718968717ad5..eb788921ff4f 100644 --- a/drivers/gpu/drm/msm/Makefile +++ b/drivers/gpu/drm/msm/Makefile @@ -1,9 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-y := -I $(srctree)/$(src) +ccflags-y := -I $(src) ccflags-y += -I $(obj)/generated -ccflags-y += -I $(srctree)/$(src)/disp/dpu1 -ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(srctree)/$(src)/dsi -ccflags-$(CONFIG_DRM_MSM_DP) += -I $(srctree)/$(src)/dp +ccflags-y += -I $(src)/disp/dpu1 +ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(src)/dsi +ccflags-$(CONFIG_DRM_MSM_DP) += -I $(src)/dp adreno-y := \ adreno/adreno_device.o \ @@ -163,8 +163,8 @@ else endif quiet_cmd_headergen = GENHDR $@ - cmd_headergen = mkdir -p $(obj)/generated && $(PYTHON3) $(srctree)/$(src)/registers/gen_header.py \ - $(headergen-opts) --rnn $(srctree)/$(src)/registers --xml $< c-defines > $@ + cmd_headergen = mkdir -p $(obj)/generated && $(PYTHON3) $(src)/registers/gen_header.py \ + $(headergen-opts) --rnn $(src)/registers --xml $< c-defines > $@ $(obj)/generated/%.xml.h: $(src)/registers/adreno/%.xml \ $(src)/registers/adreno/adreno_common.xml \ diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild index cf6b3a80c0c8..c32c01827c1d 100644 --- a/drivers/gpu/drm/nouveau/Kbuild +++ b/drivers/gpu/drm/nouveau/Kbuild @@ -1,10 +1,8 @@ -NOUVEAU_PATH ?= $(srctree) - # SPDX-License-Identifier: MIT -ccflags-y += -I $(NOUVEAU_PATH)/$(src)/include -ccflags-y += -I $(NOUVEAU_PATH)/$(src)/include/nvkm -ccflags-y += -I $(NOUVEAU_PATH)/$(src)/nvkm -ccflags-y += -I $(NOUVEAU_PATH)/$(src) +ccflags-y += -I $(src)/include +ccflags-y += -I $(src)/include/nvkm +ccflags-y += -I $(src)/nvkm +ccflags-y += -I $(src) # NVKM - HW resource manager #- code also used by various userspace tools/tests diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c index 3421e8389222..9ea0c64c26b5 100644 --- a/drivers/gpu/drm/omapdrm/omap_gem.c +++ b/drivers/gpu/drm/omapdrm/omap_gem.c @@ -9,6 +9,7 @@ #include <linux/shmem_fs.h> #include <linux/spinlock.h> #include <linux/pfn_t.h> +#include <linux/vmalloc.h> #include <drm/drm_prime.h> #include <drm/drm_vma_manager.h> diff --git a/drivers/gpu/drm/v3d/v3d_bo.c b/drivers/gpu/drm/v3d/v3d_bo.c index a07ede668cc1..a165cbcdd27b 100644 --- a/drivers/gpu/drm/v3d/v3d_bo.c +++ b/drivers/gpu/drm/v3d/v3d_bo.c @@ -21,6 +21,7 @@ #include <linux/dma-buf.h> #include <linux/pfn_t.h> +#include <linux/vmalloc.h> #include "v3d_drv.h" #include "uapi/drm/v3d_drm.h" diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c index ae2de914eb89..2731f6ded1c2 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c @@ -54,6 +54,7 @@ #include "vmwgfx_drv.h" #include "vmwgfx_binding.h" #include "device_include/svga3d_reg.h" +#include <linux/vmalloc.h> #define VMW_BINDING_RT_BIT 0 #define VMW_BINDING_PS_BIT 1 diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c index 195ff8792e5a..dd4ca6a9c690 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c @@ -31,6 +31,7 @@ #include <drm/ttm/ttm_placement.h> #include <linux/sched/signal.h> +#include <linux/vmalloc.h> bool vmw_supports_3d(struct vmw_private *dev_priv) { diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c b/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c index 829df395c2ed..6e6beff9e262 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c @@ -25,6 +25,7 @@ * **************************************************************************/ +#include <linux/vmalloc.h> #include "vmwgfx_devcaps.h" #include "vmwgfx_drv.h" diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c index bdad93864b98..8f1730aeacc9 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c @@ -54,6 +54,7 @@ #include <linux/module.h> #include <linux/pci.h> #include <linux/version.h> +#include <linux/vmalloc.h> #define VMWGFX_DRIVER_DESC "Linux drm driver for VMware graphics devices" diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index cc3086e649eb..2e52d73eba48 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -35,6 +35,7 @@ #include <linux/sync_file.h> #include <linux/hashtable.h> +#include <linux/vmalloc.h> /* * Helper macro to get dx_ctx_node if available otherwise print an error diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c index a1da5678c731..835d1eed8dd9 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c @@ -31,6 +31,7 @@ #include <drm/vmwgfx_drm.h> #include <linux/pci.h> +#include <linux/vmalloc.h> int vmw_getparam_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 8321ec4f9b46..b165bbf52aef 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -32,7 +32,7 @@ endif # Enable -Werror in CI and development subdir-ccflags-$(CONFIG_DRM_XE_WERROR) += -Werror -subdir-ccflags-y += -I$(obj) -I$(srctree)/$(src) +subdir-ccflags-y += -I$(obj) -I$(src) # generated sources hostprogs := xe_gen_wa_oob @@ -43,7 +43,7 @@ quiet_cmd_wa_oob = GEN $(notdir $(generated_oob)) cmd_wa_oob = mkdir -p $(@D); $^ $(generated_oob) $(obj)/generated/%_wa_oob.c $(obj)/generated/%_wa_oob.h: $(obj)/xe_gen_wa_oob \ - $(srctree)/$(src)/xe_wa_oob.rules + $(src)/xe_wa_oob.rules $(call cmd,wa_oob) uses_generated_oob := \ @@ -176,8 +176,8 @@ endif # i915 Display compat #defines and #includes subdir-ccflags-$(CONFIG_DRM_XE_DISPLAY) += \ - -I$(srctree)/$(src)/display/ext \ - -I$(srctree)/$(src)/compat-i915-headers \ + -I$(src)/display/ext \ + -I$(src)/compat-i915-headers \ -I$(srctree)/drivers/gpu/drm/i915/display/ \ -Ddrm_i915_gem_object=xe_bo \ -Ddrm_i915_private=xe_device @@ -319,7 +319,7 @@ ifneq ($(CONFIG_DRM_XE_DISPLAY),y) endif always-$(CONFIG_DRM_XE_WERROR) += \ - $(patsubst %.h,%.hdrtest, $(shell cd $(srctree)/$(src) && find * -name '*.h' $(hdrtest_find_args))) + $(patsubst %.h,%.hdrtest, $(shell cd $(src) && find * -name '*.h' $(hdrtest_find_args))) quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) cmd_hdrtest = $(CC) -DHDRTEST $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; touch $@ diff --git a/drivers/gpu/drm/xen/xen_drm_front_gem.c b/drivers/gpu/drm/xen/xen_drm_front_gem.c index 3ad2b4cfd1f0..63112ed975c4 100644 --- a/drivers/gpu/drm/xen/xen_drm_front_gem.c +++ b/drivers/gpu/drm/xen/xen_drm_front_gem.c @@ -11,6 +11,7 @@ #include <linux/dma-buf.h> #include <linux/scatterlist.h> #include <linux/shmem_fs.h> +#include <linux/vmalloc.h> #include <drm/drm_gem.h> #include <drm/drm_prime.h> diff --git a/drivers/hid/amd-sfh-hid/Makefile b/drivers/hid/amd-sfh-hid/Makefile index 0222170ab7ad..106514b54d16 100644 --- a/drivers/hid/amd-sfh-hid/Makefile +++ b/drivers/hid/amd-sfh-hid/Makefile @@ -13,4 +13,4 @@ amd_sfh-objs += sfh1_1/amd_sfh_init.o amd_sfh-objs += sfh1_1/amd_sfh_interface.o amd_sfh-objs += sfh1_1/amd_sfh_desc.o -ccflags-y += -I $(srctree)/$(src)/ +ccflags-y += -I $(src)/ diff --git a/drivers/hid/intel-ish-hid/Makefile b/drivers/hid/intel-ish-hid/Makefile index a927b224cd44..e1e062e4b542 100644 --- a/drivers/hid/intel-ish-hid/Makefile +++ b/drivers/hid/intel-ish-hid/Makefile @@ -24,4 +24,4 @@ intel-ishtp-hid-objs += ishtp-hid-client.o obj-$(CONFIG_INTEL_ISH_FIRMWARE_DOWNLOADER) += intel-ishtp-loader.o intel-ishtp-loader-objs += ishtp-fw-loader.o -ccflags-y += -I $(srctree)/$(src)/ishtp +ccflags-y += -I $(src)/ishtp diff --git a/drivers/hsi/controllers/omap_ssi_core.c b/drivers/hsi/controllers/omap_ssi_core.c index 6802efb4d6cd..3140990a6164 100644 --- a/drivers/hsi/controllers/omap_ssi_core.c +++ b/drivers/hsi/controllers/omap_ssi_core.c @@ -547,7 +547,7 @@ out1: return err; } -static int ssi_remove(struct platform_device *pd) +static void ssi_remove(struct platform_device *pd) { struct hsi_controller *ssi = platform_get_drvdata(pd); @@ -561,8 +561,6 @@ static int ssi_remove(struct platform_device *pd) platform_set_drvdata(pd, NULL); pm_runtime_disable(&pd->dev); - - return 0; } #ifdef CONFIG_PM @@ -618,7 +616,7 @@ MODULE_DEVICE_TABLE(of, omap_ssi_of_match); static struct platform_driver ssi_pdriver = { .probe = ssi_probe, - .remove = ssi_remove, + .remove_new = ssi_remove, .driver = { .name = "omap_ssi", .pm = DEV_PM_OPS, diff --git a/drivers/hsi/controllers/omap_ssi_port.c b/drivers/hsi/controllers/omap_ssi_port.c index c78d9c9f7371..f0b3eca7376e 100644 --- a/drivers/hsi/controllers/omap_ssi_port.c +++ b/drivers/hsi/controllers/omap_ssi_port.c @@ -1224,7 +1224,7 @@ error: return err; } -static int ssi_port_remove(struct platform_device *pd) +static void ssi_port_remove(struct platform_device *pd) { struct hsi_port *port = platform_get_drvdata(pd); struct omap_ssi_port *omap_port = hsi_port_drvdata(port); @@ -1251,8 +1251,6 @@ static int ssi_port_remove(struct platform_device *pd) pm_runtime_dont_use_autosuspend(&pd->dev); pm_runtime_disable(&pd->dev); - - return 0; } static int ssi_restore_divisor(struct omap_ssi_port *omap_port) @@ -1387,7 +1385,7 @@ MODULE_DEVICE_TABLE(of, omap_ssi_port_of_match); struct platform_driver ssi_port_pdriver = { .probe = ssi_port_probe, - .remove = ssi_port_remove, + .remove_new = ssi_port_remove, .driver = { .name = "omap_ssi_port", .of_match_table = omap_ssi_port_of_match, diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c index dde3f9b6871a..9c452bfbd571 100644 --- a/drivers/hv/hv_common.c +++ b/drivers/hv/hv_common.c @@ -561,11 +561,7 @@ EXPORT_SYMBOL_GPL(hv_query_ext_cap); void hv_setup_dma_ops(struct device *dev, bool coherent) { - /* - * Hyper-V does not offer a vIOMMU in the guest - * VM, so pass 0/NULL for the IOMMU settings - */ - arch_setup_dma_ops(dev, 0, 0, coherent); + arch_setup_dma_ops(dev, coherent); } EXPORT_SYMBOL_GPL(hv_setup_dma_ops); diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c index 3949ded0d4fa..375bd0d89b0c 100644 --- a/drivers/hwtracing/coresight/coresight-catu.c +++ b/drivers/hwtracing/coresight/coresight-catu.c @@ -590,7 +590,6 @@ MODULE_DEVICE_TABLE(amba, catu_ids); static struct amba_driver catu_driver = { .drv = { .name = "coresight-catu", - .owner = THIS_MODULE, .suppress_bind_attrs = true, }, .probe = catu_probe, diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index e805617020d0..d2b5a5718c29 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -982,7 +982,6 @@ MODULE_DEVICE_TABLE(amba, cti_ids); static struct amba_driver cti_driver = { .drv = { .name = "coresight-cti", - .owner = THIS_MODULE, .suppress_bind_attrs = true, }, .probe = cti_probe, diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index 3aab182b562f..7edd3f1d0d46 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -844,7 +844,6 @@ MODULE_DEVICE_TABLE(amba, etb_ids); static struct amba_driver etb_driver = { .drv = { .name = "coresight-etb10", - .owner = THIS_MODULE, .pm = &etb_dev_pm_ops, .suppress_bind_attrs = true, diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index 9d5c1391ffb1..8b362605d242 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -1008,7 +1008,6 @@ MODULE_DEVICE_TABLE(amba, etm_ids); static struct amba_driver etm_driver = { .drv = { .name = "coresight-etm3x", - .owner = THIS_MODULE, .pm = &etm_dev_pm_ops, .suppress_bind_attrs = true, }, diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index c2ca4a02dfce..e6cd9705596c 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -2344,7 +2344,6 @@ MODULE_DEVICE_TABLE(amba, etm4_ids); static struct amba_driver etm4x_amba_driver = { .drv = { .name = "coresight-etm4x", - .owner = THIS_MODULE, .suppress_bind_attrs = true, }, .probe = etm4_probe_amba, diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c index ef1a0abfee4e..5ab1f592917a 100644 --- a/drivers/hwtracing/coresight/coresight-funnel.c +++ b/drivers/hwtracing/coresight/coresight-funnel.c @@ -399,7 +399,6 @@ MODULE_DEVICE_TABLE(amba, dynamic_funnel_ids); static struct amba_driver dynamic_funnel_driver = { .drv = { .name = "coresight-dynamic-funnel", - .owner = THIS_MODULE, .pm = &funnel_dev_pm_ops, .suppress_bind_attrs = true, }, diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c index 73452d9dc13b..2bb9ba66e3c0 100644 --- a/drivers/hwtracing/coresight/coresight-replicator.c +++ b/drivers/hwtracing/coresight/coresight-replicator.c @@ -406,7 +406,6 @@ static struct amba_driver dynamic_replicator_driver = { .drv = { .name = "coresight-dynamic-replicator", .pm = &replicator_dev_pm_ops, - .owner = THIS_MODULE, .suppress_bind_attrs = true, }, .probe = dynamic_replicator_probe, diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index 974d37e5f94c..15b52358965c 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -954,7 +954,6 @@ MODULE_DEVICE_TABLE(amba, stm_ids); static struct amba_driver stm_driver = { .drv = { .name = "coresight-stm", - .owner = THIS_MODULE, .pm = &stm_dev_pm_ops, .suppress_bind_attrs = true, }, diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index 72005b0c633e..0d251cae814f 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -602,7 +602,6 @@ MODULE_DEVICE_TABLE(amba, tmc_ids); static struct amba_driver tmc_driver = { .drv = { .name = "coresight-tmc", - .owner = THIS_MODULE, .suppress_bind_attrs = true, }, .probe = tmc_probe, diff --git a/drivers/hwtracing/coresight/coresight-tpda.c b/drivers/hwtracing/coresight/coresight-tpda.c index 7739bc7adc44..bfca103f9f84 100644 --- a/drivers/hwtracing/coresight/coresight-tpda.c +++ b/drivers/hwtracing/coresight/coresight-tpda.c @@ -333,7 +333,6 @@ static struct amba_id tpda_ids[] = { static struct amba_driver tpda_driver = { .drv = { .name = "coresight-tpda", - .owner = THIS_MODULE, .suppress_bind_attrs = true, }, .probe = tpda_probe, diff --git a/drivers/hwtracing/coresight/coresight-tpdm.c b/drivers/hwtracing/coresight/coresight-tpdm.c index a9708ab0d488..0726f8842552 100644 --- a/drivers/hwtracing/coresight/coresight-tpdm.c +++ b/drivers/hwtracing/coresight/coresight-tpdm.c @@ -1310,7 +1310,6 @@ static struct amba_id tpdm_ids[] = { static struct amba_driver tpdm_driver = { .drv = { .name = "coresight-tpdm", - .owner = THIS_MODULE, .suppress_bind_attrs = true, }, .probe = tpdm_probe, diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c index 29024f880fda..7dc9ea564bca 100644 --- a/drivers/hwtracing/coresight/coresight-tpiu.c +++ b/drivers/hwtracing/coresight/coresight-tpiu.c @@ -236,7 +236,6 @@ MODULE_DEVICE_TABLE(amba, tpiu_ids); static struct amba_driver tpiu_driver = { .drv = { .name = "coresight-tpiu", - .owner = THIS_MODULE, .pm = &tpiu_dev_pm_ops, .suppress_bind_attrs = true, }, diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 6136776482e6..96a32b213669 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -17,6 +17,7 @@ #include <asm/barrier.h> #include <asm/cpufeature.h> +#include <linux/vmalloc.h> #include "coresight-self-hosted-trace.h" #include "coresight-trbe.h" diff --git a/drivers/hwtracing/intel_th/core.c b/drivers/hwtracing/intel_th/core.c index cc7f879bb175..86c8efecd7c2 100644 --- a/drivers/hwtracing/intel_th/core.c +++ b/drivers/hwtracing/intel_th/core.c @@ -871,7 +871,7 @@ intel_th_alloc(struct device *dev, const struct intel_th_drvdata *drvdata, if (!th) return ERR_PTR(-ENOMEM); - th->id = ida_simple_get(&intel_th_ida, 0, 0, GFP_KERNEL); + th->id = ida_alloc(&intel_th_ida, GFP_KERNEL); if (th->id < 0) { err = th->id; goto err_alloc; @@ -931,7 +931,7 @@ err_chrdev: "intel_th/output"); err_ida: - ida_simple_remove(&intel_th_ida, th->id); + ida_free(&intel_th_ida, th->id); err_alloc: kfree(th); @@ -964,7 +964,7 @@ void intel_th_free(struct intel_th *th) __unregister_chrdev(th->major, 0, TH_POSSIBLE_OUTPUTS, "intel_th/output"); - ida_simple_remove(&intel_th_ida, th->id); + ida_free(&intel_th_ida, th->id); kfree(th); } diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 97989c914260..fe6e8a1bb607 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -18,7 +18,7 @@ config I2C_CCGX_UCSI config I2C_ALI1535 tristate "ALI 1535" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the SMB Host controller on Acer Labs Inc. (ALI) M1535 South Bridges. The SMB @@ -30,7 +30,7 @@ config I2C_ALI1535 config I2C_ALI1563 tristate "ALI 1563" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the SMB Host controller on Acer Labs Inc. (ALI) M1563 South Bridges. The SMB @@ -42,7 +42,7 @@ config I2C_ALI1563 config I2C_ALI15X3 tristate "ALI 15x3" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the Acer Labs Inc. (ALI) M1514 and M1543 motherboard I2C interfaces. @@ -52,7 +52,7 @@ config I2C_ALI15X3 config I2C_AMD756 tristate "AMD 756/766/768/8111 and nVidia nForce" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the AMD 756/766/768 mainboard I2C interfaces. The driver also includes @@ -77,7 +77,7 @@ config I2C_AMD756_S4882 config I2C_AMD8111 tristate "AMD 8111" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the second (SMBus 2.0) AMD 8111 mainboard I2C interface. @@ -107,7 +107,7 @@ config I2C_HIX5HD2 config I2C_I801 tristate "Intel 82801 (ICH/PCH)" - depends on PCI + depends on PCI && HAS_IOPORT select P2SB if X86 select CHECK_SIGNATURE if X86 && DMI select I2C_SMBUS @@ -163,9 +163,17 @@ config I2C_I801 This driver can also be built as a module. If so, the module will be called i2c-i801. +config I2C_I801_MUX + def_bool I2C_I801 + depends on DMI && I2C_MUX_GPIO + depends on !(I2C_I801=y && I2C_MUX=m) + help + Optional support for multiplexed SMBUS on certain systems with + more than 8 memory slots. + config I2C_ISCH tristate "Intel SCH SMBus 1.0" - depends on PCI + depends on PCI && HAS_IOPORT select LPC_SCH help Say Y here if you want to use SMBus controller on the Intel SCH @@ -186,7 +194,7 @@ config I2C_ISMT config I2C_PIIX4 tristate "Intel PIIX4 and compatible (ATI/AMD/Serverworks/Broadcom/SMSC)" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the Intel PIIX4 family of mainboard I2C interfaces. Specifically, the following @@ -232,7 +240,7 @@ config I2C_CHT_WC config I2C_NFORCE2 tristate "Nvidia nForce2, nForce3 and nForce4" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the Nvidia nForce2, nForce3 and nForce4 families of mainboard I2C interfaces. @@ -265,7 +273,7 @@ config I2C_NVIDIA_GPU config I2C_SIS5595 tristate "SiS 5595" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the SiS5595 SMBus (a subset of I2C) interface. @@ -275,7 +283,7 @@ config I2C_SIS5595 config I2C_SIS630 tristate "SiS 630/730/964" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the SiS630, SiS730 and SiS964 SMBus (a subset of I2C) interface. @@ -285,7 +293,7 @@ config I2C_SIS630 config I2C_SIS96X tristate "SiS 96x" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the SiS 96x SMBus (a subset of I2C) interfaces. Specifically, the following @@ -303,7 +311,7 @@ config I2C_SIS96X config I2C_VIA tristate "VIA VT82C586B" - depends on PCI + depends on PCI && HAS_IOPORT select I2C_ALGOBIT help If you say yes to this option, support will be included for the VIA @@ -314,7 +322,7 @@ config I2C_VIA config I2C_VIAPRO tristate "VIA VT82C596/82C686/82xx and CX700/VX8xx/VX900" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the VIA VT82C596 and later SMBus interface. Specifically, the following @@ -336,6 +344,16 @@ config I2C_VIAPRO if ACPI +config I2C_ZHAOXIN + tristate "Zhaoxin I2C Interface" + depends on PCI || COMPILE_TEST + help + If you say yes to this option, support will be included for the + ZHAOXIN I2C interface + + This driver can also be built as a module. If so, the module + will be called i2c-zhaoxin. + comment "ACPI drivers" config I2C_SCMI @@ -500,7 +518,7 @@ config I2C_BRCMSTB config I2C_CADENCE tristate "Cadence I2C Controller" - depends on ARCH_ZYNQ || ARM64 || XTENSA || COMPILE_TEST + depends on ARCH_ZYNQ || ARM64 || XTENSA || RISCV || COMPILE_TEST help Say yes here to select Cadence I2C Host Controller. This controller is e.g. used by Xilinx Zynq. @@ -1397,6 +1415,7 @@ config I2C_ICY config I2C_MLXCPLD tristate "Mellanox I2C driver" depends on X86_64 || (ARM64 && ACPI) || COMPILE_TEST + depends on HAS_IOPORT help This exposes the Mellanox platform I2C busses to the linux I2C layer for X86 and ARM64/ACPI based systems. diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index aa0ee8ecd6f2..3d65934f5eb4 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -29,6 +29,8 @@ obj-$(CONFIG_I2C_SIS630) += i2c-sis630.o obj-$(CONFIG_I2C_SIS96X) += i2c-sis96x.o obj-$(CONFIG_I2C_VIA) += i2c-via.o obj-$(CONFIG_I2C_VIAPRO) += i2c-viapro.o +i2c-zhaoxin-objs := i2c-viai2c-zhaoxin.o i2c-viai2c-common.o +obj-$(CONFIG_I2C_ZHAOXIN) += i2c-zhaoxin.o # Mac SMBus host controller drivers obj-$(CONFIG_I2C_HYDRA) += i2c-hydra.o @@ -118,6 +120,7 @@ obj-$(CONFIG_I2C_TEGRA_BPMP) += i2c-tegra-bpmp.o obj-$(CONFIG_I2C_UNIPHIER) += i2c-uniphier.o obj-$(CONFIG_I2C_UNIPHIER_F) += i2c-uniphier-f.o obj-$(CONFIG_I2C_VERSATILE) += i2c-versatile.o +i2c-wmt-objs := i2c-viai2c-wmt.o i2c-viai2c-common.o obj-$(CONFIG_I2C_WMT) += i2c-wmt.o i2c-octeon-objs := i2c-octeon-core.o i2c-octeon-platdrv.o obj-$(CONFIG_I2C_OCTEON) += i2c-octeon.o diff --git a/drivers/i2c/busses/i2c-ali1535.c b/drivers/i2c/busses/i2c-ali1535.c index 461eb23f9d47..9d7b4efe26ad 100644 --- a/drivers/i2c/busses/i2c-ali1535.c +++ b/drivers/i2c/busses/i2c-ali1535.c @@ -285,10 +285,8 @@ static int ali1535_transaction(struct i2c_adapter *adap) && (timeout++ < MAX_TIMEOUT)); /* If the SMBus is still busy, we give up */ - if (timeout > MAX_TIMEOUT) { + if (timeout > MAX_TIMEOUT) result = -ETIMEDOUT; - dev_err(&adap->dev, "SMBus Timeout!\n"); - } if (temp & ALI1535_STS_FAIL) { result = -EIO; @@ -313,10 +311,8 @@ static int ali1535_transaction(struct i2c_adapter *adap) } /* check to see if the "command complete" indication is set */ - if (!(temp & ALI1535_STS_DONE)) { + if (!(temp & ALI1535_STS_DONE)) result = -ETIMEDOUT; - dev_err(&adap->dev, "Error: command never completed\n"); - } dev_dbg(&adap->dev, "Transaction (post): STS=%02x, TYP=%02x, " "CMD=%02x, ADD=%02x, DAT0=%02x, DAT1=%02x\n", diff --git a/drivers/i2c/busses/i2c-ali1563.c b/drivers/i2c/busses/i2c-ali1563.c index 307fb0666ecb..63897a89bb35 100644 --- a/drivers/i2c/busses/i2c-ali1563.c +++ b/drivers/i2c/busses/i2c-ali1563.c @@ -99,7 +99,6 @@ static int ali1563_transaction(struct i2c_adapter *a, int size) return 0; if (!timeout) { - dev_err(&a->dev, "Timeout - Trying to KILL transaction!\n"); /* Issue 'kill' to host controller */ outb_p(HST_CNTL2_KILL, SMB_HST_CNTL2); data = inb_p(SMB_HST_STS); diff --git a/drivers/i2c/busses/i2c-ali15x3.c b/drivers/i2c/busses/i2c-ali15x3.c index d2fa30deb054..956e5020d71e 100644 --- a/drivers/i2c/busses/i2c-ali15x3.c +++ b/drivers/i2c/busses/i2c-ali15x3.c @@ -294,10 +294,8 @@ static int ali15x3_transaction(struct i2c_adapter *adap) && (timeout++ < MAX_TIMEOUT)); /* If the SMBus is still busy, we give up */ - if (timeout > MAX_TIMEOUT) { + if (timeout > MAX_TIMEOUT) result = -ETIMEDOUT; - dev_err(&adap->dev, "SMBus Timeout!\n"); - } if (temp & ALI15X3_STS_TERM) { result = -EIO; diff --git a/drivers/i2c/busses/i2c-amd-mp2-plat.c b/drivers/i2c/busses/i2c-amd-mp2-plat.c index 112fe2bc5662..d3ac1c77a509 100644 --- a/drivers/i2c/busses/i2c-amd-mp2-plat.c +++ b/drivers/i2c/busses/i2c-amd-mp2-plat.c @@ -97,17 +97,17 @@ static void i2c_amd_cmd_completion(struct amd_i2c_common *i2c_common) static int i2c_amd_check_cmd_completion(struct amd_i2c_dev *i2c_dev) { struct amd_i2c_common *i2c_common = &i2c_dev->common; - unsigned long timeout; + unsigned long time_left; - timeout = wait_for_completion_timeout(&i2c_dev->cmd_complete, - i2c_dev->adap.timeout); + time_left = wait_for_completion_timeout(&i2c_dev->cmd_complete, + i2c_dev->adap.timeout); if ((i2c_common->reqcmd == i2c_read || i2c_common->reqcmd == i2c_write) && i2c_common->msg->len > 32) i2c_amd_dma_unmap(i2c_common); - if (timeout == 0) { + if (time_left == 0) { amd_mp2_rw_timeout(i2c_common); return -ETIMEDOUT; } diff --git a/drivers/i2c/busses/i2c-at91-master.c b/drivers/i2c/busses/i2c-at91-master.c index d311981d3e60..ee3b469ddfb9 100644 --- a/drivers/i2c/busses/i2c-at91-master.c +++ b/drivers/i2c/busses/i2c-at91-master.c @@ -591,7 +591,6 @@ static int at91_do_twi_transfer(struct at91_twi_dev *dev) dev->adapter.timeout); if (time_left == 0) { dev->transfer_status |= at91_twi_read(dev, AT91_TWI_SR); - dev_err(dev->dev, "controller timed out\n"); at91_init_twi_bus(dev); ret = -ETIMEDOUT; goto error; diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c index e905734c26a0..133d02899c6b 100644 --- a/drivers/i2c/busses/i2c-bcm-iproc.c +++ b/drivers/i2c/busses/i2c-bcm-iproc.c @@ -811,8 +811,6 @@ static int bcm_iproc_i2c_xfer_wait(struct bcm_iproc_i2c_dev *iproc_i2c, } if (!time_left && !iproc_i2c->xfer_is_done) { - dev_err(iproc_i2c->device, "transaction timed out\n"); - /* flush both TX/RX FIFOs */ val = BIT(M_FIFO_RX_FLUSH_SHIFT) | BIT(M_FIFO_TX_FLUSH_SHIFT); iproc_i2c_wr_reg(iproc_i2c, M_FIFO_CTRL_OFFSET, val); diff --git a/drivers/i2c/busses/i2c-bcm2835.c b/drivers/i2c/busses/i2c-bcm2835.c index b92de1944221..3045ba82380d 100644 --- a/drivers/i2c/busses/i2c-bcm2835.c +++ b/drivers/i2c/busses/i2c-bcm2835.c @@ -370,7 +370,6 @@ static int bcm2835_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], if (!time_left) { bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_C, BCM2835_I2C_C_CLEAR); - dev_err(i2c_dev->dev, "i2c transfer timed out\n"); return -ETIMEDOUT; } diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c index 4bb7d6756947..87b9ba95b2e1 100644 --- a/drivers/i2c/busses/i2c-cadence.c +++ b/drivers/i2c/busses/i2c-cadence.c @@ -633,6 +633,7 @@ static void cdns_i2c_mrecv(struct cdns_i2c *id) if (hold_clear) { ctrl_reg &= ~CDNS_I2C_CR_HOLD; + ctrl_reg &= ~CDNS_I2C_CR_CLR_FIFO; /* * In case of Xilinx Zynq SOC, clear the HOLD bit before transfer size * register reaches '0'. This is an IP bug which causes transfer size @@ -789,8 +790,6 @@ static int cdns_i2c_process_msg(struct cdns_i2c *id, struct i2c_msg *msg, time_left = wait_for_completion_timeout(&id->xfer_done, msg_timeout); if (time_left == 0) { cdns_i2c_master_reset(adap); - dev_err(id->adap.dev.parent, - "timeout waiting on completion\n"); return -ETIMEDOUT; } diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c index 02b3b1160fb0..7ae611120cfa 100644 --- a/drivers/i2c/busses/i2c-davinci.c +++ b/drivers/i2c/busses/i2c-davinci.c @@ -489,7 +489,6 @@ i2c_davinci_xfer_msg(struct i2c_adapter *adap, struct i2c_msg *msg, int stop) time_left = wait_for_completion_timeout(&dev->cmd_complete, dev->adapter.timeout); if (!time_left) { - dev_err(dev->dev, "controller timed out\n"); i2c_recover_bus(adap); dev->buf_len = 0; return -ETIMEDOUT; diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c index 9be9a2658e1f..a1b379a1e904 100644 --- a/drivers/i2c/busses/i2c-designware-pcidrv.c +++ b/drivers/i2c/busses/i2c-designware-pcidrv.c @@ -424,8 +424,6 @@ static struct pci_driver dw_i2c_driver = { }; module_pci_driver(dw_i2c_driver); -/* Work with hotplug and coldplug */ -MODULE_ALIAS("i2c_designware-pci"); MODULE_AUTHOR("Baruch Siach <baruch@tkos.co.il>"); MODULE_DESCRIPTION("Synopsys DesignWare PCI I2C bus adapter"); MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index 4ab41ba39d55..29aac9c87368 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -46,6 +46,7 @@ static const struct acpi_device_id dw_i2c_acpi_match[] = { { "INT33C3", 0 }, { "INT3432", 0 }, { "INT3433", 0 }, + { "INTC10EF", 0 }, { "80860F41", ACCESS_NO_IRQ_SUSPEND }, { "808622C1", ACCESS_NO_IRQ_SUSPEND }, { "AMD0010", ACCESS_INTR_MASK }, @@ -479,8 +480,11 @@ static const struct dev_pm_ops dw_i2c_dev_pm_ops = { RUNTIME_PM_OPS(dw_i2c_plat_runtime_suspend, dw_i2c_plat_runtime_resume, NULL) }; -/* Work with hotplug and coldplug */ -MODULE_ALIAS("platform:i2c_designware"); +static const struct platform_device_id dw_i2c_platform_ids[] = { + { "i2c_designware" }, + {} +}; +MODULE_DEVICE_TABLE(platform, dw_i2c_platform_ids); static struct platform_driver dw_i2c_driver = { .probe = dw_i2c_plat_probe, @@ -491,6 +495,7 @@ static struct platform_driver dw_i2c_driver = { .acpi_match_table = ACPI_PTR(dw_i2c_acpi_match), .pm = pm_ptr(&dw_i2c_dev_pm_ops), }, + .id_table = dw_i2c_platform_ids, }; static int __init dw_i2c_init_driver(void) diff --git a/drivers/i2c/busses/i2c-digicolor.c b/drivers/i2c/busses/i2c-digicolor.c index 3462f2bc0fa8..737604ae11fc 100644 --- a/drivers/i2c/busses/i2c-digicolor.c +++ b/drivers/i2c/busses/i2c-digicolor.c @@ -213,7 +213,7 @@ out: static int dc_i2c_xfer_msg(struct dc_i2c *i2c, struct i2c_msg *msg, int first, int last) { - unsigned long timeout = msecs_to_jiffies(TIMEOUT_MS); + unsigned long time_left = msecs_to_jiffies(TIMEOUT_MS); unsigned long flags; spin_lock_irqsave(&i2c->lock, flags); @@ -227,9 +227,9 @@ static int dc_i2c_xfer_msg(struct dc_i2c *i2c, struct i2c_msg *msg, int first, dc_i2c_start_msg(i2c, first); spin_unlock_irqrestore(&i2c->lock, flags); - timeout = wait_for_completion_timeout(&i2c->done, timeout); + time_left = wait_for_completion_timeout(&i2c->done, time_left); dc_i2c_set_irq(i2c, 0); - if (timeout == 0) { + if (time_left == 0) { i2c->state = STATE_IDLE; return -ETIMEDOUT; } diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c index 385ef9d9e4d4..d8baca9b610c 100644 --- a/drivers/i2c/busses/i2c-exynos5.c +++ b/drivers/i2c/busses/i2c-exynos5.c @@ -763,7 +763,7 @@ static bool exynos5_i2c_poll_irqs_timeout(struct exynos5_i2c *i2c, static int exynos5_i2c_xfer_msg(struct exynos5_i2c *i2c, struct i2c_msg *msgs, int stop) { - unsigned long timeout; + unsigned long time_left; int ret; i2c->msg = msgs; @@ -775,13 +775,13 @@ static int exynos5_i2c_xfer_msg(struct exynos5_i2c *i2c, exynos5_i2c_message_start(i2c, stop); if (!i2c->atomic) - timeout = wait_for_completion_timeout(&i2c->msg_complete, - EXYNOS5_I2C_TIMEOUT); - else - timeout = exynos5_i2c_poll_irqs_timeout(i2c, + time_left = wait_for_completion_timeout(&i2c->msg_complete, EXYNOS5_I2C_TIMEOUT); + else + time_left = exynos5_i2c_poll_irqs_timeout(i2c, + EXYNOS5_I2C_TIMEOUT); - if (timeout == 0) + if (time_left == 0) ret = -ETIMEDOUT; else ret = i2c->state; diff --git a/drivers/i2c/busses/i2c-hix5hd2.c b/drivers/i2c/busses/i2c-hix5hd2.c index 8e75515c3ca4..a47b9939fa2c 100644 --- a/drivers/i2c/busses/i2c-hix5hd2.c +++ b/drivers/i2c/busses/i2c-hix5hd2.c @@ -314,7 +314,7 @@ static void hix5hd2_i2c_message_start(struct hix5hd2_i2c_priv *priv, int stop) static int hix5hd2_i2c_xfer_msg(struct hix5hd2_i2c_priv *priv, struct i2c_msg *msgs, int stop) { - unsigned long timeout; + unsigned long time_left; int ret; priv->msg = msgs; @@ -327,9 +327,9 @@ static int hix5hd2_i2c_xfer_msg(struct hix5hd2_i2c_priv *priv, reinit_completion(&priv->msg_complete); hix5hd2_i2c_message_start(priv, stop); - timeout = wait_for_completion_timeout(&priv->msg_complete, - priv->adap.timeout); - if (timeout == 0) { + time_left = wait_for_completion_timeout(&priv->msg_complete, + priv->adap.timeout); + if (time_left == 0) { priv->state = HIX5I2C_STAT_RW_ERR; priv->err = -ETIMEDOUT; dev_warn(priv->dev, "%s timeout=%d\n", diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 79870dd7a014..d2d2a6dbe29f 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -105,6 +105,7 @@ #include <linux/ioport.h> #include <linux/init.h> #include <linux/i2c.h> +#include <linux/i2c-mux.h> #include <linux/i2c-smbus.h> #include <linux/acpi.h> #include <linux/io.h> @@ -119,7 +120,7 @@ #include <linux/pm_runtime.h> #include <linux/mutex.h> -#if IS_ENABLED(CONFIG_I2C_MUX_GPIO) && defined CONFIG_DMI +#ifdef CONFIG_I2C_I801_MUX #include <linux/gpio/machine.h> #include <linux/platform_data/i2c-mux-gpio.h> #endif @@ -263,7 +264,6 @@ struct i801_mux_config { char *gpio_chip; unsigned values[3]; int n_values; - unsigned classes[3]; unsigned gpios[2]; /* Relative to gpio_chip->base */ int n_gpios; }; @@ -288,9 +288,10 @@ struct i801_priv { int len; u8 *data; -#if IS_ENABLED(CONFIG_I2C_MUX_GPIO) && defined CONFIG_DMI +#ifdef CONFIG_I2C_I801_MUX struct platform_device *mux_pdev; struct gpiod_lookup_table *lookup; + struct notifier_block mux_notifier_block; #endif struct platform_device *tco_pdev; @@ -398,9 +399,7 @@ static int i801_check_post(struct i801_priv *priv, int status) * If the SMBus is still busy, we give up */ if (unlikely(status < 0)) { - dev_err(&priv->pci_dev->dev, "Transaction timeout\n"); /* try to stop the current command */ - dev_dbg(&priv->pci_dev->dev, "Terminating the current operation\n"); outb_p(SMBHSTCNT_KILL, SMBHSTCNT(priv)); usleep_range(1000, 2000); outb_p(0, SMBHSTCNT(priv)); @@ -409,7 +408,7 @@ static int i801_check_post(struct i801_priv *priv, int status) status = inb_p(SMBHSTSTS(priv)); if ((status & SMBHSTSTS_HOST_BUSY) || !(status & SMBHSTSTS_FAILED)) - dev_err(&priv->pci_dev->dev, + dev_dbg(&priv->pci_dev->dev, "Failed terminating the transaction\n"); return -ETIMEDOUT; } @@ -1059,7 +1058,7 @@ static const struct pci_device_id i801_ids[] = { MODULE_DEVICE_TABLE(pci, i801_ids); #if defined CONFIG_X86 && defined CONFIG_DMI -static unsigned char apanel_addr; +static unsigned char apanel_addr __ro_after_init; /* Scan the system ROM for the signature "FJKEYINF" */ static __init const void __iomem *bios_signature(const void __iomem *bios) @@ -1298,7 +1297,7 @@ static void i801_probe_optional_slaves(struct i801_priv *priv) register_dell_lis3lv02d_i2c_device(priv); /* Instantiate SPD EEPROMs unless the SMBus is multiplexed */ -#if IS_ENABLED(CONFIG_I2C_MUX_GPIO) +#ifdef CONFIG_I2C_I801_MUX if (!priv->mux_pdev) #endif i2c_register_spd(&priv->adapter); @@ -1308,12 +1307,11 @@ static void __init input_apanel_init(void) {} static void i801_probe_optional_slaves(struct i801_priv *priv) {} #endif /* CONFIG_X86 && CONFIG_DMI */ -#if IS_ENABLED(CONFIG_I2C_MUX_GPIO) && defined CONFIG_DMI +#ifdef CONFIG_I2C_I801_MUX static struct i801_mux_config i801_mux_config_asus_z8_d12 = { .gpio_chip = "gpio_ich", .values = { 0x02, 0x03 }, .n_values = 2, - .classes = { I2C_CLASS_SPD, I2C_CLASS_SPD }, .gpios = { 52, 53 }, .n_gpios = 2, }; @@ -1322,7 +1320,6 @@ static struct i801_mux_config i801_mux_config_asus_z8_d18 = { .gpio_chip = "gpio_ich", .values = { 0x02, 0x03, 0x01 }, .n_values = 3, - .classes = { I2C_CLASS_SPD, I2C_CLASS_SPD, I2C_CLASS_SPD }, .gpios = { 52, 53 }, .n_gpios = 2, }; @@ -1394,6 +1391,23 @@ static const struct dmi_system_id mux_dmi_table[] = { { } }; +static int i801_notifier_call(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct i801_priv *priv = container_of(nb, struct i801_priv, mux_notifier_block); + struct device *dev = data; + + if (action != BUS_NOTIFY_ADD_DEVICE || + dev->type != &i2c_adapter_type || + i2c_root_adapter(dev) != &priv->adapter) + return NOTIFY_DONE; + + /* Call i2c_register_spd for muxed child segments */ + i2c_register_spd(to_i2c_adapter(dev)); + + return NOTIFY_OK; +} + /* Setup multiplexing if needed */ static void i801_add_mux(struct i801_priv *priv) { @@ -1415,7 +1429,6 @@ static void i801_add_mux(struct i801_priv *priv) gpio_data.parent = priv->adapter.nr; gpio_data.values = mux_config->values; gpio_data.n_values = mux_config->n_values; - gpio_data.classes = mux_config->classes; gpio_data.idle = I2C_MUX_GPIO_NO_IDLE; /* Register GPIO descriptor lookup table */ @@ -1430,6 +1443,9 @@ static void i801_add_mux(struct i801_priv *priv) mux_config->gpios[i], "mux", 0); gpiod_add_lookup_table(lookup); + priv->mux_notifier_block.notifier_call = i801_notifier_call; + if (bus_register_notifier(&i2c_bus_type, &priv->mux_notifier_block)) + return; /* * Register the mux device, we use PLATFORM_DEVID_NONE here * because since we are referring to the GPIO chip by name we are @@ -1451,6 +1467,7 @@ static void i801_add_mux(struct i801_priv *priv) static void i801_del_mux(struct i801_priv *priv) { + bus_unregister_notifier(&i2c_bus_type, &priv->mux_notifier_block); platform_device_unregister(priv->mux_pdev); gpiod_remove_lookup_table(priv->lookup); } diff --git a/drivers/i2c/busses/i2c-img-scb.c b/drivers/i2c/busses/i2c-img-scb.c index f9d4bfef511c..e0e87185f6bb 100644 --- a/drivers/i2c/busses/i2c-img-scb.c +++ b/drivers/i2c/busses/i2c-img-scb.c @@ -1124,11 +1124,8 @@ static int img_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, IMG_I2C_TIMEOUT); del_timer_sync(&i2c->check_timer); - if (time_left == 0) { - dev_err(adap->dev.parent, "i2c transfer timed out\n"); + if (time_left == 0) i2c->msg_status = -ETIMEDOUT; - break; - } if (i2c->msg_status) break; diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index 6d72e4e126dd..0197786892a2 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -99,6 +99,7 @@ struct lpi2c_imx_struct { __u8 *rx_buf; __u8 *tx_buf; struct completion complete; + unsigned long rate_per; unsigned int msglen; unsigned int delivered; unsigned int block_data; @@ -212,9 +213,7 @@ static int lpi2c_imx_config(struct lpi2c_imx_struct *lpi2c_imx) lpi2c_imx_set_mode(lpi2c_imx); - clk_rate = clk_get_rate(lpi2c_imx->clks[0].clk); - if (!clk_rate) - return -EINVAL; + clk_rate = lpi2c_imx->rate_per; if (lpi2c_imx->mode == HS || lpi2c_imx->mode == ULTRA_FAST) filt = 0; @@ -308,11 +307,11 @@ static int lpi2c_imx_master_disable(struct lpi2c_imx_struct *lpi2c_imx) static int lpi2c_imx_msg_complete(struct lpi2c_imx_struct *lpi2c_imx) { - unsigned long timeout; + unsigned long time_left; - timeout = wait_for_completion_timeout(&lpi2c_imx->complete, HZ); + time_left = wait_for_completion_timeout(&lpi2c_imx->complete, HZ); - return timeout ? 0 : -ETIMEDOUT; + return time_left ? 0 : -ETIMEDOUT; } static int lpi2c_imx_txfifo_empty(struct lpi2c_imx_struct *lpi2c_imx) @@ -611,6 +610,20 @@ static int lpi2c_imx_probe(struct platform_device *pdev) if (ret) return ret; + /* + * Lock the parent clock rate to avoid getting parent clock upon + * each transfer + */ + ret = devm_clk_rate_exclusive_get(&pdev->dev, lpi2c_imx->clks[0].clk); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "can't lock I2C peripheral clock rate\n"); + + lpi2c_imx->rate_per = clk_get_rate(lpi2c_imx->clks[0].clk); + if (!lpi2c_imx->rate_per) + return dev_err_probe(&pdev->dev, -EINVAL, + "can't get I2C peripheral clock rate\n"); + pm_runtime_set_autosuspend_delay(&pdev->dev, I2C_PM_TIMEOUT); pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_get_noresume(&pdev->dev); diff --git a/drivers/i2c/busses/i2c-ismt.c b/drivers/i2c/busses/i2c-ismt.c index c74985d77b0e..655b5d851c48 100644 --- a/drivers/i2c/busses/i2c-ismt.c +++ b/drivers/i2c/busses/i2c-ismt.c @@ -623,7 +623,6 @@ static int ismt_access(struct i2c_adapter *adap, u16 addr, dma_unmap_single(dev, dma_addr, dma_size, dma_direction); if (unlikely(!time_left)) { - dev_err(dev, "completion wait timed out\n"); ret = -ETIMEDOUT; goto out; } diff --git a/drivers/i2c/busses/i2c-jz4780.c b/drivers/i2c/busses/i2c-jz4780.c index 55035cca0ae5..7951891d6b97 100644 --- a/drivers/i2c/busses/i2c-jz4780.c +++ b/drivers/i2c/busses/i2c-jz4780.c @@ -565,7 +565,7 @@ static inline int jz4780_i2c_xfer_read(struct jz4780_i2c *i2c, int idx) { int ret = 0; - long timeout; + unsigned long time_left; int wait_time = JZ4780_I2C_TIMEOUT * (len + 5); unsigned short tmp; unsigned long flags; @@ -600,10 +600,10 @@ static inline int jz4780_i2c_xfer_read(struct jz4780_i2c *i2c, spin_unlock_irqrestore(&i2c->lock, flags); - timeout = wait_for_completion_timeout(&i2c->trans_waitq, - msecs_to_jiffies(wait_time)); + time_left = wait_for_completion_timeout(&i2c->trans_waitq, + msecs_to_jiffies(wait_time)); - if (!timeout) { + if (!time_left) { dev_err(&i2c->adap.dev, "irq read timeout\n"); dev_dbg(&i2c->adap.dev, "send cmd count:%d %d\n", i2c->cmd, i2c->cmd_buf[i2c->cmd]); @@ -627,7 +627,7 @@ static inline int jz4780_i2c_xfer_write(struct jz4780_i2c *i2c, { int ret = 0; int wait_time = JZ4780_I2C_TIMEOUT * (len + 5); - long timeout; + unsigned long time_left; unsigned short tmp; unsigned long flags; @@ -655,14 +655,14 @@ static inline int jz4780_i2c_xfer_write(struct jz4780_i2c *i2c, spin_unlock_irqrestore(&i2c->lock, flags); - timeout = wait_for_completion_timeout(&i2c->trans_waitq, - msecs_to_jiffies(wait_time)); - if (timeout && !i2c->stop_hold) { + time_left = wait_for_completion_timeout(&i2c->trans_waitq, + msecs_to_jiffies(wait_time)); + if (time_left && !i2c->stop_hold) { unsigned short i2c_sta; int write_in_process; - timeout = JZ4780_I2C_TIMEOUT * 100; - for (; timeout > 0; timeout--) { + time_left = JZ4780_I2C_TIMEOUT * 100; + for (; time_left > 0; time_left--) { i2c_sta = jz4780_i2c_readw(i2c, JZ4780_I2C_STA); write_in_process = (i2c_sta & JZ4780_I2C_STA_MSTACT) || @@ -673,7 +673,7 @@ static inline int jz4780_i2c_xfer_write(struct jz4780_i2c *i2c, } } - if (!timeout) { + if (!time_left) { dev_err(&i2c->adap.dev, "write wait timeout\n"); ret = -EIO; } diff --git a/drivers/i2c/busses/i2c-mpc.c b/drivers/i2c/busses/i2c-mpc.c index 8d73c0f405ed..c4223556b3b8 100644 --- a/drivers/i2c/busses/i2c-mpc.c +++ b/drivers/i2c/busses/i2c-mpc.c @@ -304,13 +304,12 @@ static void mpc_i2c_setup_512x(struct device_node *node, struct mpc_i2c *i2c, u32 clock) { - struct device_node *node_ctrl; void __iomem *ctrl; u32 idx; /* Enable I2C interrupts for mpc5121 */ - node_ctrl = of_find_compatible_node(NULL, NULL, - "fsl,mpc5121-i2c-ctrl"); + struct device_node *node_ctrl __free(device_node) = + of_find_compatible_node(NULL, NULL, "fsl,mpc5121-i2c-ctrl"); if (node_ctrl) { ctrl = of_iomap(node_ctrl, 0); if (ctrl) { @@ -321,7 +320,6 @@ static void mpc_i2c_setup_512x(struct device_node *node, setbits32(ctrl, 1 << (24 + idx * 2)); iounmap(ctrl); } - of_node_put(node_ctrl); } /* The clock setup for the 52xx works also fine for the 512x */ @@ -358,11 +356,11 @@ static const struct mpc_i2c_divider mpc_i2c_dividers_8xxx[] = { static u32 mpc_i2c_get_sec_cfg_8xxx(void) { - struct device_node *node; u32 __iomem *reg; u32 val = 0; - node = of_find_node_by_name(NULL, "global-utilities"); + struct device_node *node __free(device_node) = + of_find_node_by_name(NULL, "global-utilities"); if (node) { const u32 *prop = of_get_property(node, "reg", NULL); if (prop) { @@ -383,7 +381,6 @@ static u32 mpc_i2c_get_sec_cfg_8xxx(void) iounmap(reg); } } - of_node_put(node); return val; } diff --git a/drivers/i2c/busses/i2c-nomadik.c b/drivers/i2c/busses/i2c-nomadik.c index 4f41a3c7824d..ad0f02acdb12 100644 --- a/drivers/i2c/busses/i2c-nomadik.c +++ b/drivers/i2c/busses/i2c-nomadik.c @@ -542,12 +542,9 @@ static int read_i2c(struct nmk_i2c_dev *priv, u16 flags) xfer_done = nmk_i2c_wait_xfer_done(priv); - if (!xfer_done) { - /* Controller timed out */ - dev_err(&priv->adev->dev, "read from slave 0x%x timed out\n", - priv->cli.slave_adr); + if (!xfer_done) status = -ETIMEDOUT; - } + return status; } @@ -1194,7 +1191,6 @@ MODULE_DEVICE_TABLE(amba, nmk_i2c_ids); static struct amba_driver nmk_i2c_driver = { .drv = { - .owner = THIS_MODULE, .name = DRIVER_NAME, .pm = pm_ptr(&nmk_i2c_pm), }, diff --git a/drivers/i2c/busses/i2c-ocores.c b/drivers/i2c/busses/i2c-ocores.c index e106af83cef4..56a4dabf5a38 100644 --- a/drivers/i2c/busses/i2c-ocores.c +++ b/drivers/i2c/busses/i2c-ocores.c @@ -32,7 +32,6 @@ */ struct ocores_i2c { void __iomem *base; - int iobase; u32 reg_shift; u32 reg_io_width; unsigned long flags; @@ -136,16 +135,6 @@ static inline u8 oc_getreg_32be(struct ocores_i2c *i2c, int reg) return ioread32be(i2c->base + (reg << i2c->reg_shift)); } -static void oc_setreg_io_8(struct ocores_i2c *i2c, int reg, u8 value) -{ - outb(value, i2c->iobase + reg); -} - -static inline u8 oc_getreg_io_8(struct ocores_i2c *i2c, int reg) -{ - return inb(i2c->iobase + reg); -} - static inline void oc_setreg(struct ocores_i2c *i2c, int reg, u8 value) { i2c->setreg(i2c, reg, value); @@ -618,15 +607,19 @@ static int ocores_i2c_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_IO, 0); if (!res) return -EINVAL; - i2c->iobase = res->start; if (!devm_request_region(&pdev->dev, res->start, resource_size(res), pdev->name)) { dev_err(&pdev->dev, "Can't get I/O resource.\n"); return -EBUSY; } - i2c->setreg = oc_setreg_io_8; - i2c->getreg = oc_getreg_io_8; + i2c->base = devm_ioport_map(&pdev->dev, res->start, + resource_size(res)); + if (!i2c->base) { + dev_err(&pdev->dev, "Can't map I/O resource.\n"); + return -EBUSY; + } + i2c->reg_io_width = 1; } pdata = dev_get_platdata(&pdev->dev); diff --git a/drivers/i2c/busses/i2c-octeon-core.c b/drivers/i2c/busses/i2c-octeon-core.c index 845eda70b8ca..5b7b942141e7 100644 --- a/drivers/i2c/busses/i2c-octeon-core.c +++ b/drivers/i2c/busses/i2c-octeon-core.c @@ -17,9 +17,14 @@ #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/pci.h> #include "i2c-octeon-core.h" +#define INITIAL_DELTA_HZ 1000000 +#define TWSI_MASTER_CLK_REG_DEF_VAL 0x18 +#define TWSI_MASTER_CLK_REG_OTX2_VAL 0x3 + /* interrupt service routine */ irqreturn_t octeon_i2c_isr(int irq, void *dev_id) { @@ -80,7 +85,7 @@ static int octeon_i2c_wait(struct octeon_i2c *i2c) static bool octeon_i2c_hlc_test_valid(struct octeon_i2c *i2c) { - return (__raw_readq(i2c->twsi_base + SW_TWSI(i2c)) & SW_TWSI_V) == 0; + return (__raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)) & SW_TWSI_V) == 0; } static void octeon_i2c_hlc_int_clear(struct octeon_i2c *i2c) @@ -177,13 +182,14 @@ static int octeon_i2c_hlc_wait(struct octeon_i2c *i2c) static int octeon_i2c_check_status(struct octeon_i2c *i2c, int final_read) { u8 stat; + u64 mode; /* * This is ugly... in HLC mode the status is not in the status register - * but in the lower 8 bits of SW_TWSI. + * but in the lower 8 bits of OCTEON_REG_SW_TWSI. */ if (i2c->hlc_enabled) - stat = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + stat = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); else stat = octeon_i2c_stat_read(i2c); @@ -239,6 +245,13 @@ static int octeon_i2c_check_status(struct octeon_i2c *i2c, int final_read) case STAT_RXADDR_NAK: case STAT_AD2W_NAK: return -ENXIO; + + case STAT_WDOG_TOUT: + mode = __raw_readq(i2c->twsi_base + OCTEON_REG_MODE(i2c)); + /* Set BUS_MON_RST to reset bus monitor */ + mode |= BUS_MON_RST_MASK; + octeon_i2c_writeq_flush(mode, i2c->twsi_base + OCTEON_REG_MODE(i2c)); + return -EIO; default: dev_err(i2c->dev, "unhandled state: %d\n", stat); return -EIO; @@ -419,12 +432,12 @@ static int octeon_i2c_hlc_read(struct octeon_i2c *i2c, struct i2c_msg *msgs) else cmd |= SW_TWSI_OP_7; - octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c)); + octeon_i2c_writeq_flush(cmd, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); ret = octeon_i2c_hlc_wait(i2c); if (ret) goto err; - cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if ((cmd & SW_TWSI_R) == 0) return octeon_i2c_check_status(i2c, false); @@ -432,7 +445,7 @@ static int octeon_i2c_hlc_read(struct octeon_i2c *i2c, struct i2c_msg *msgs) msgs[0].buf[j] = (cmd >> (8 * i)) & 0xff; if (msgs[0].len > 4) { - cmd = __raw_readq(i2c->twsi_base + SW_TWSI_EXT(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI_EXT(i2c)); for (i = 0; i < msgs[0].len - 4 && i < 4; i++, j--) msgs[0].buf[j] = (cmd >> (8 * i)) & 0xff; } @@ -469,15 +482,15 @@ static int octeon_i2c_hlc_write(struct octeon_i2c *i2c, struct i2c_msg *msgs) for (i = 0; i < msgs[0].len - 4 && i < 4; i++, j--) ext |= (u64)msgs[0].buf[j] << (8 * i); - octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT(i2c)); + octeon_i2c_writeq_flush(ext, i2c->twsi_base + OCTEON_REG_SW_TWSI_EXT(i2c)); } - octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c)); + octeon_i2c_writeq_flush(cmd, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); ret = octeon_i2c_hlc_wait(i2c); if (ret) goto err; - cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if ((cmd & SW_TWSI_R) == 0) return octeon_i2c_check_status(i2c, false); @@ -510,19 +523,19 @@ static int octeon_i2c_hlc_comp_read(struct octeon_i2c *i2c, struct i2c_msg *msgs cmd |= SW_TWSI_EIA; ext = (u64)msgs[0].buf[0] << SW_TWSI_IA_SHIFT; cmd |= (u64)msgs[0].buf[1] << SW_TWSI_IA_SHIFT; - octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT(i2c)); + octeon_i2c_writeq_flush(ext, i2c->twsi_base + OCTEON_REG_SW_TWSI_EXT(i2c)); } else { cmd |= (u64)msgs[0].buf[0] << SW_TWSI_IA_SHIFT; } octeon_i2c_hlc_int_clear(i2c); - octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c)); + octeon_i2c_writeq_flush(cmd, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); ret = octeon_i2c_hlc_wait(i2c); if (ret) goto err; - cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if ((cmd & SW_TWSI_R) == 0) return octeon_i2c_check_status(i2c, false); @@ -530,7 +543,7 @@ static int octeon_i2c_hlc_comp_read(struct octeon_i2c *i2c, struct i2c_msg *msgs msgs[1].buf[j] = (cmd >> (8 * i)) & 0xff; if (msgs[1].len > 4) { - cmd = __raw_readq(i2c->twsi_base + SW_TWSI_EXT(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI_EXT(i2c)); for (i = 0; i < msgs[1].len - 4 && i < 4; i++, j--) msgs[1].buf[j] = (cmd >> (8 * i)) & 0xff; } @@ -577,16 +590,16 @@ static int octeon_i2c_hlc_comp_write(struct octeon_i2c *i2c, struct i2c_msg *msg set_ext = true; } if (set_ext) - octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT(i2c)); + octeon_i2c_writeq_flush(ext, i2c->twsi_base + OCTEON_REG_SW_TWSI_EXT(i2c)); octeon_i2c_hlc_int_clear(i2c); - octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c)); + octeon_i2c_writeq_flush(cmd, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); ret = octeon_i2c_hlc_wait(i2c); if (ret) goto err; - cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if ((cmd & SW_TWSI_R) == 0) return octeon_i2c_check_status(i2c, false); @@ -607,25 +620,27 @@ int octeon_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num) struct octeon_i2c *i2c = i2c_get_adapdata(adap); int i, ret = 0; - if (num == 1) { - if (msgs[0].len > 0 && msgs[0].len <= 8) { - if (msgs[0].flags & I2C_M_RD) - ret = octeon_i2c_hlc_read(i2c, msgs); - else - ret = octeon_i2c_hlc_write(i2c, msgs); - goto out; - } - } else if (num == 2) { - if ((msgs[0].flags & I2C_M_RD) == 0 && - (msgs[1].flags & I2C_M_RECV_LEN) == 0 && - msgs[0].len > 0 && msgs[0].len <= 2 && - msgs[1].len > 0 && msgs[1].len <= 8 && - msgs[0].addr == msgs[1].addr) { - if (msgs[1].flags & I2C_M_RD) - ret = octeon_i2c_hlc_comp_read(i2c, msgs); - else - ret = octeon_i2c_hlc_comp_write(i2c, msgs); - goto out; + if (IS_LS_FREQ(i2c->twsi_freq)) { + if (num == 1) { + if (msgs[0].len > 0 && msgs[0].len <= 8) { + if (msgs[0].flags & I2C_M_RD) + ret = octeon_i2c_hlc_read(i2c, msgs); + else + ret = octeon_i2c_hlc_write(i2c, msgs); + goto out; + } + } else if (num == 2) { + if ((msgs[0].flags & I2C_M_RD) == 0 && + (msgs[1].flags & I2C_M_RECV_LEN) == 0 && + msgs[0].len > 0 && msgs[0].len <= 2 && + msgs[1].len > 0 && msgs[1].len <= 8 && + msgs[0].addr == msgs[1].addr) { + if (msgs[1].flags & I2C_M_RD) + ret = octeon_i2c_hlc_comp_read(i2c, msgs); + else + ret = octeon_i2c_hlc_comp_write(i2c, msgs); + goto out; + } } } @@ -658,31 +673,64 @@ out: void octeon_i2c_set_clock(struct octeon_i2c *i2c) { int tclk, thp_base, inc, thp_idx, mdiv_idx, ndiv_idx, foscl, diff; - int thp = 0x18, mdiv = 2, ndiv = 0, delta_hz = 1000000; + bool is_plat_otx2; + /* + * Find divisors to produce target frequency, start with large delta + * to cover wider range of divisors, note thp = TCLK half period and + * ds is OSCL output frequency divisor. + */ + unsigned int thp, mdiv_min, mdiv = 2, ndiv = 0, ds = 10; + unsigned int delta_hz = INITIAL_DELTA_HZ; + + is_plat_otx2 = octeon_i2c_is_otx2(to_pci_dev(i2c->dev)); + + if (is_plat_otx2) { + thp = TWSI_MASTER_CLK_REG_OTX2_VAL; + mdiv_min = 0; + if (!IS_LS_FREQ(i2c->twsi_freq)) + ds = 15; + } else { + thp = TWSI_MASTER_CLK_REG_DEF_VAL; + mdiv_min = 2; + } for (ndiv_idx = 0; ndiv_idx < 8 && delta_hz != 0; ndiv_idx++) { /* * An mdiv value of less than 2 seems to not work well * with ds1337 RTCs, so we constrain it to larger values. */ - for (mdiv_idx = 15; mdiv_idx >= 2 && delta_hz != 0; mdiv_idx--) { + for (mdiv_idx = 15; mdiv_idx >= mdiv_min && delta_hz != 0; mdiv_idx--) { /* * For given ndiv and mdiv values check the * two closest thp values. */ - tclk = i2c->twsi_freq * (mdiv_idx + 1) * 10; + tclk = i2c->twsi_freq * (mdiv_idx + 1) * ds; tclk *= (1 << ndiv_idx); - thp_base = (i2c->sys_freq / (tclk * 2)) - 1; + if (is_plat_otx2) + thp_base = (i2c->sys_freq / tclk) - 2; + else + thp_base = (i2c->sys_freq / (tclk * 2)) - 1; for (inc = 0; inc <= 1; inc++) { thp_idx = thp_base + inc; if (thp_idx < 5 || thp_idx > 0xff) continue; - foscl = i2c->sys_freq / (2 * (thp_idx + 1)); + if (is_plat_otx2) + foscl = i2c->sys_freq / (thp_idx + 2); + else + foscl = i2c->sys_freq / + (2 * (thp_idx + 1)); foscl = foscl / (1 << ndiv_idx); - foscl = foscl / (mdiv_idx + 1) / 10; + foscl = foscl / (mdiv_idx + 1) / ds; + if (foscl > i2c->twsi_freq) + continue; diff = abs(foscl - i2c->twsi_freq); + /* + * Diff holds difference between calculated frequency + * value vs desired frequency. + * Delta_hz is updated with last minimum diff. + */ if (diff < delta_hz) { delta_hz = diff; thp = thp_idx; @@ -694,6 +742,17 @@ void octeon_i2c_set_clock(struct octeon_i2c *i2c) } octeon_i2c_reg_write(i2c, SW_TWSI_OP_TWSI_CLK, thp); octeon_i2c_reg_write(i2c, SW_TWSI_EOP_TWSI_CLKCTL, (mdiv << 3) | ndiv); + if (is_plat_otx2) { + u64 mode; + + mode = __raw_readq(i2c->twsi_base + OCTEON_REG_MODE(i2c)); + /* Set REFCLK_SRC and HS_MODE in TWSX_MODE register */ + if (!IS_LS_FREQ(i2c->twsi_freq)) + mode |= TWSX_MODE_HS_MASK; + else + mode &= ~TWSX_MODE_HS_MASK; + octeon_i2c_writeq_flush(mode, i2c->twsi_base + OCTEON_REG_MODE(i2c)); + } } int octeon_i2c_init_lowlevel(struct octeon_i2c *i2c) diff --git a/drivers/i2c/busses/i2c-octeon-core.h b/drivers/i2c/busses/i2c-octeon-core.h index 9bb9f64fdda0..7af01864da75 100644 --- a/drivers/i2c/busses/i2c-octeon-core.h +++ b/drivers/i2c/busses/i2c-octeon-core.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <linux/atomic.h> +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/delay.h> #include <linux/device.h> @@ -7,6 +8,7 @@ #include <linux/i2c-smbus.h> #include <linux/io.h> #include <linux/kernel.h> +#include <linux/pci.h> /* Controller command patterns */ #define SW_TWSI_V BIT_ULL(63) /* Valid bit */ @@ -71,6 +73,7 @@ #define STAT_SLAVE_ACK 0xC8 #define STAT_AD2W_ACK 0xD0 #define STAT_AD2W_NAK 0xD8 +#define STAT_WDOG_TOUT 0xF0 #define STAT_IDLE 0xF8 /* TWSI_INT values */ @@ -92,11 +95,21 @@ struct octeon_i2c_reg_offset { unsigned int sw_twsi; unsigned int twsi_int; unsigned int sw_twsi_ext; + unsigned int mode; }; -#define SW_TWSI(x) (x->roff.sw_twsi) -#define TWSI_INT(x) (x->roff.twsi_int) -#define SW_TWSI_EXT(x) (x->roff.sw_twsi_ext) +#define OCTEON_REG_SW_TWSI(x) ((x)->roff.sw_twsi) +#define OCTEON_REG_TWSI_INT(x) ((x)->roff.twsi_int) +#define OCTEON_REG_SW_TWSI_EXT(x) ((x)->roff.sw_twsi_ext) +#define OCTEON_REG_MODE(x) ((x)->roff.mode) + +/* Set REFCLK_SRC and HS_MODE in TWSX_MODE register */ +#define TWSX_MODE_REFCLK_SRC BIT(4) +#define TWSX_MODE_HS_MODE BIT(0) +#define TWSX_MODE_HS_MASK (TWSX_MODE_REFCLK_SRC | TWSX_MODE_HS_MODE) + +/* Set BUS_MON_RST to reset bus monitor */ +#define BUS_MON_RST_MASK BIT(3) struct octeon_i2c { wait_queue_head_t queue; @@ -134,16 +147,16 @@ static inline void octeon_i2c_writeq_flush(u64 val, void __iomem *addr) * @eop_reg: Register selector * @data: Value to be written * - * The I2C core registers are accessed indirectly via the SW_TWSI CSR. + * The I2C core registers are accessed indirectly via the OCTEON_REG_SW_TWSI CSR. */ static inline void octeon_i2c_reg_write(struct octeon_i2c *i2c, u64 eop_reg, u8 data) { int tries = 1000; u64 tmp; - __raw_writeq(SW_TWSI_V | eop_reg | data, i2c->twsi_base + SW_TWSI(i2c)); + __raw_writeq(SW_TWSI_V | eop_reg | data, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); do { - tmp = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + tmp = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if (--tries < 0) return; } while ((tmp & SW_TWSI_V) != 0); @@ -169,9 +182,9 @@ static inline int octeon_i2c_reg_read(struct octeon_i2c *i2c, u64 eop_reg, int tries = 1000; u64 tmp; - __raw_writeq(SW_TWSI_V | eop_reg | SW_TWSI_R, i2c->twsi_base + SW_TWSI(i2c)); + __raw_writeq(SW_TWSI_V | eop_reg | SW_TWSI_R, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); do { - tmp = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + tmp = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if (--tries < 0) { /* signal that the returned data is invalid */ if (error) @@ -191,24 +204,40 @@ static inline int octeon_i2c_reg_read(struct octeon_i2c *i2c, u64 eop_reg, octeon_i2c_reg_read(i2c, SW_TWSI_EOP_TWSI_STAT, NULL) /** - * octeon_i2c_read_int - read the TWSI_INT register + * octeon_i2c_read_int - read the OCTEON_REG_TWSI_INT register * @i2c: The struct octeon_i2c * * Returns the value of the register. */ static inline u64 octeon_i2c_read_int(struct octeon_i2c *i2c) { - return __raw_readq(i2c->twsi_base + TWSI_INT(i2c)); + return __raw_readq(i2c->twsi_base + OCTEON_REG_TWSI_INT(i2c)); } /** - * octeon_i2c_write_int - write the TWSI_INT register + * octeon_i2c_write_int - write the OCTEON_REG_TWSI_INT register * @i2c: The struct octeon_i2c * @data: Value to be written */ static inline void octeon_i2c_write_int(struct octeon_i2c *i2c, u64 data) { - octeon_i2c_writeq_flush(data, i2c->twsi_base + TWSI_INT(i2c)); + octeon_i2c_writeq_flush(data, i2c->twsi_base + OCTEON_REG_TWSI_INT(i2c)); +} + +#define IS_LS_FREQ(twsi_freq) ((twsi_freq) <= 400000) +#define PCI_SUBSYS_DEVID_9XXX 0xB +#define PCI_SUBSYS_MASK GENMASK(15, 12) +/** + * octeon_i2c_is_otx2 - check for chip ID + * @pdev: PCI dev structure + * + * Returns true if the device is an OcteonTX2, false otherwise. + */ +static inline bool octeon_i2c_is_otx2(struct pci_dev *pdev) +{ + u32 chip_id = FIELD_GET(PCI_SUBSYS_MASK, pdev->subsystem_device); + + return (chip_id == PCI_SUBSYS_DEVID_9XXX); } /* Prototypes */ diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c index 42165ef57946..30a5ea282a8b 100644 --- a/drivers/i2c/busses/i2c-omap.c +++ b/drivers/i2c/busses/i2c-omap.c @@ -660,7 +660,7 @@ static int omap_i2c_xfer_msg(struct i2c_adapter *adap, struct i2c_msg *msg, int stop, bool polling) { struct omap_i2c_dev *omap = i2c_get_adapdata(adap); - unsigned long timeout; + unsigned long time_left; u16 w; int ret; @@ -740,19 +740,18 @@ static int omap_i2c_xfer_msg(struct i2c_adapter *adap, * into arbitration and we're currently unable to recover from it. */ if (!polling) { - timeout = wait_for_completion_timeout(&omap->cmd_complete, - OMAP_I2C_TIMEOUT); + time_left = wait_for_completion_timeout(&omap->cmd_complete, + OMAP_I2C_TIMEOUT); } else { do { omap_i2c_wait(omap); ret = omap_i2c_xfer_data(omap); } while (ret == -EAGAIN); - timeout = !ret; + time_left = !ret; } - if (timeout == 0) { - dev_err(omap->dev, "controller timed out\n"); + if (time_left == 0) { omap_i2c_reset(omap); __omap_i2c_init(omap); return -ETIMEDOUT; diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c index 888ca636f3f3..f495560bd99c 100644 --- a/drivers/i2c/busses/i2c-pxa.c +++ b/drivers/i2c/busses/i2c-pxa.c @@ -826,7 +826,7 @@ static inline void i2c_pxa_stop_message(struct pxa_i2c *i2c) static int i2c_pxa_send_mastercode(struct pxa_i2c *i2c) { u32 icr; - long timeout; + long time_left; spin_lock_irq(&i2c->lock); i2c->highmode_enter = true; @@ -837,12 +837,12 @@ static int i2c_pxa_send_mastercode(struct pxa_i2c *i2c) writel(icr, _ICR(i2c)); spin_unlock_irq(&i2c->lock); - timeout = wait_event_timeout(i2c->wait, - i2c->highmode_enter == false, HZ * 1); + time_left = wait_event_timeout(i2c->wait, + i2c->highmode_enter == false, HZ * 1); i2c->highmode_enter = false; - return (timeout == 0) ? I2C_RETRY : 0; + return (time_left == 0) ? I2C_RETRY : 0; } /* @@ -1050,7 +1050,7 @@ static irqreturn_t i2c_pxa_handler(int this_irq, void *dev_id) */ static int i2c_pxa_do_xfer(struct pxa_i2c *i2c, struct i2c_msg *msg, int num) { - long timeout; + long time_left; int ret; /* @@ -1095,7 +1095,7 @@ static int i2c_pxa_do_xfer(struct pxa_i2c *i2c, struct i2c_msg *msg, int num) /* * The rest of the processing occurs in the interrupt handler. */ - timeout = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); + time_left = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); i2c_pxa_stop_message(i2c); /* @@ -1103,7 +1103,7 @@ static int i2c_pxa_do_xfer(struct pxa_i2c *i2c, struct i2c_msg *msg, int num) */ ret = i2c->msg_idx; - if (!timeout && i2c->msg_num) { + if (!time_left && i2c->msg_num) { i2c_pxa_scream_blue_murder(i2c, "timeout with active message"); i2c_recover_bus(&i2c->adap); ret = I2C_RETRY; diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index da94df466e83..0a8b95ce35f7 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -586,7 +586,8 @@ static int geni_i2c_gpi_xfer(struct geni_i2c_dev *gi2c, struct i2c_msg msgs[], i { struct dma_slave_config config = {}; struct gpi_i2c_config peripheral = {}; - int i, ret = 0, timeout; + int i, ret = 0; + unsigned long time_left; dma_addr_t tx_addr, rx_addr; void *tx_buf = NULL, *rx_buf = NULL; const struct geni_i2c_clk_fld *itr = gi2c->clk_fld; @@ -629,12 +630,9 @@ static int geni_i2c_gpi_xfer(struct geni_i2c_dev *gi2c, struct i2c_msg msgs[], i dma_async_issue_pending(gi2c->tx_c); - timeout = wait_for_completion_timeout(&gi2c->done, XFER_TIMEOUT); - if (!timeout) { - dev_err(gi2c->se.dev, "I2C timeout gpi flags:%d addr:0x%x\n", - gi2c->cur->flags, gi2c->cur->addr); + time_left = wait_for_completion_timeout(&gi2c->done, XFER_TIMEOUT); + if (!time_left) gi2c->err = -ETIMEDOUT; - } if (gi2c->err) { ret = gi2c->err; diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c index 598102d16677..c9b43a3c4bd3 100644 --- a/drivers/i2c/busses/i2c-qup.c +++ b/drivers/i2c/busses/i2c-qup.c @@ -793,10 +793,8 @@ static int qup_i2c_bam_schedule_desc(struct qup_i2c_dev *qup) dma_async_issue_pending(qup->brx.dma); } - if (!wait_for_completion_timeout(&qup->xfer, qup->xfer_timeout)) { - dev_err(qup->dev, "normal trans timed out\n"); + if (!wait_for_completion_timeout(&qup->xfer, qup->xfer_timeout)) ret = -ETIMEDOUT; - } if (ret || qup->bus_err || qup->qup_err) { reinit_completion(&qup->xfer); diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c index e43ff483c56e..f608b1838cad 100644 --- a/drivers/i2c/busses/i2c-riic.c +++ b/drivers/i2c/busses/i2c-riic.c @@ -46,18 +46,6 @@ #include <linux/pm_runtime.h> #include <linux/reset.h> -#define RIIC_ICCR1 0x00 -#define RIIC_ICCR2 0x04 -#define RIIC_ICMR1 0x08 -#define RIIC_ICMR3 0x10 -#define RIIC_ICSER 0x18 -#define RIIC_ICIER 0x1c -#define RIIC_ICSR2 0x24 -#define RIIC_ICBRL 0x34 -#define RIIC_ICBRH 0x38 -#define RIIC_ICDRT 0x3c -#define RIIC_ICDRR 0x40 - #define ICCR1_ICE 0x80 #define ICCR1_IICRST 0x40 #define ICCR1_SOWP 0x10 @@ -87,6 +75,25 @@ #define RIIC_INIT_MSG -1 +enum riic_reg_list { + RIIC_ICCR1 = 0, + RIIC_ICCR2, + RIIC_ICMR1, + RIIC_ICMR3, + RIIC_ICSER, + RIIC_ICIER, + RIIC_ICSR2, + RIIC_ICBRL, + RIIC_ICBRH, + RIIC_ICDRT, + RIIC_ICDRR, + RIIC_REG_END, +}; + +struct riic_of_data { + u8 regs[RIIC_REG_END]; +}; + struct riic_dev { void __iomem *base; u8 *buf; @@ -94,6 +101,7 @@ struct riic_dev { int bytes_left; int err; int is_last; + const struct riic_of_data *info; struct completion msg_done; struct i2c_adapter adapter; struct clk *clk; @@ -105,9 +113,19 @@ struct riic_irq_desc { char *name; }; +static inline void riic_writeb(struct riic_dev *riic, u8 val, u8 offset) +{ + writeb(val, riic->base + riic->info->regs[offset]); +} + +static inline u8 riic_readb(struct riic_dev *riic, u8 offset) +{ + return readb(riic->base + riic->info->regs[offset]); +} + static inline void riic_clear_set_bit(struct riic_dev *riic, u8 clear, u8 set, u8 reg) { - writeb((readb(riic->base + reg) & ~clear) | set, riic->base + reg); + riic_writeb(riic, (riic_readb(riic, reg) & ~clear) | set, reg); } static int riic_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) @@ -119,7 +137,7 @@ static int riic_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) pm_runtime_get_sync(adap->dev.parent); - if (readb(riic->base + RIIC_ICCR2) & ICCR2_BBSY) { + if (riic_readb(riic, RIIC_ICCR2) & ICCR2_BBSY) { riic->err = -EBUSY; goto out; } @@ -127,7 +145,7 @@ static int riic_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) reinit_completion(&riic->msg_done); riic->err = 0; - writeb(0, riic->base + RIIC_ICSR2); + riic_writeb(riic, 0, RIIC_ICSR2); for (i = 0, start_bit = ICCR2_ST; i < num; i++) { riic->bytes_left = RIIC_INIT_MSG; @@ -135,9 +153,9 @@ static int riic_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) riic->msg = &msgs[i]; riic->is_last = (i == num - 1); - writeb(ICIER_NAKIE | ICIER_TIE, riic->base + RIIC_ICIER); + riic_writeb(riic, ICIER_NAKIE | ICIER_TIE, RIIC_ICIER); - writeb(start_bit, riic->base + RIIC_ICCR2); + riic_writeb(riic, start_bit, RIIC_ICCR2); time_left = wait_for_completion_timeout(&riic->msg_done, riic->adapter.timeout); if (time_left == 0) @@ -191,7 +209,7 @@ static irqreturn_t riic_tdre_isr(int irq, void *data) * value could be moved to the shadow shift register right away. So * this must be after updates to ICIER (where we want to disable TIE)! */ - writeb(val, riic->base + RIIC_ICDRT); + riic_writeb(riic, val, RIIC_ICDRT); return IRQ_HANDLED; } @@ -200,9 +218,9 @@ static irqreturn_t riic_tend_isr(int irq, void *data) { struct riic_dev *riic = data; - if (readb(riic->base + RIIC_ICSR2) & ICSR2_NACKF) { + if (riic_readb(riic, RIIC_ICSR2) & ICSR2_NACKF) { /* We got a NACKIE */ - readb(riic->base + RIIC_ICDRR); /* dummy read */ + riic_readb(riic, RIIC_ICDRR); /* dummy read */ riic_clear_set_bit(riic, ICSR2_NACKF, 0, RIIC_ICSR2); riic->err = -ENXIO; } else if (riic->bytes_left) { @@ -211,7 +229,7 @@ static irqreturn_t riic_tend_isr(int irq, void *data) if (riic->is_last || riic->err) { riic_clear_set_bit(riic, ICIER_TEIE, ICIER_SPIE, RIIC_ICIER); - writeb(ICCR2_SP, riic->base + RIIC_ICCR2); + riic_writeb(riic, ICCR2_SP, RIIC_ICCR2); } else { /* Transfer is complete, but do not send STOP */ riic_clear_set_bit(riic, ICIER_TEIE, 0, RIIC_ICIER); @@ -230,7 +248,7 @@ static irqreturn_t riic_rdrf_isr(int irq, void *data) if (riic->bytes_left == RIIC_INIT_MSG) { riic->bytes_left = riic->msg->len; - readb(riic->base + RIIC_ICDRR); /* dummy read */ + riic_readb(riic, RIIC_ICDRR); /* dummy read */ return IRQ_HANDLED; } @@ -238,7 +256,7 @@ static irqreturn_t riic_rdrf_isr(int irq, void *data) /* STOP must come before we set ACKBT! */ if (riic->is_last) { riic_clear_set_bit(riic, 0, ICIER_SPIE, RIIC_ICIER); - writeb(ICCR2_SP, riic->base + RIIC_ICCR2); + riic_writeb(riic, ICCR2_SP, RIIC_ICCR2); } riic_clear_set_bit(riic, 0, ICMR3_ACKBT, RIIC_ICMR3); @@ -248,7 +266,7 @@ static irqreturn_t riic_rdrf_isr(int irq, void *data) } /* Reading acks the RIE interrupt */ - *riic->buf = readb(riic->base + RIIC_ICDRR); + *riic->buf = riic_readb(riic, RIIC_ICDRR); riic->buf++; riic->bytes_left--; @@ -260,10 +278,10 @@ static irqreturn_t riic_stop_isr(int irq, void *data) struct riic_dev *riic = data; /* read back registers to confirm writes have fully propagated */ - writeb(0, riic->base + RIIC_ICSR2); - readb(riic->base + RIIC_ICSR2); - writeb(0, riic->base + RIIC_ICIER); - readb(riic->base + RIIC_ICIER); + riic_writeb(riic, 0, RIIC_ICSR2); + riic_readb(riic, RIIC_ICSR2); + riic_writeb(riic, 0, RIIC_ICIER); + riic_readb(riic, RIIC_ICIER); complete(&riic->msg_done); @@ -365,15 +383,15 @@ static int riic_init_hw(struct riic_dev *riic, struct i2c_timings *t) t->scl_rise_ns / (1000000000 / rate), cks, brl, brh); /* Changing the order of accessing IICRST and ICE may break things! */ - writeb(ICCR1_IICRST | ICCR1_SOWP, riic->base + RIIC_ICCR1); + riic_writeb(riic, ICCR1_IICRST | ICCR1_SOWP, RIIC_ICCR1); riic_clear_set_bit(riic, 0, ICCR1_ICE, RIIC_ICCR1); - writeb(ICMR1_CKS(cks), riic->base + RIIC_ICMR1); - writeb(brh | ICBR_RESERVED, riic->base + RIIC_ICBRH); - writeb(brl | ICBR_RESERVED, riic->base + RIIC_ICBRL); + riic_writeb(riic, ICMR1_CKS(cks), RIIC_ICMR1); + riic_writeb(riic, brh | ICBR_RESERVED, RIIC_ICBRH); + riic_writeb(riic, brl | ICBR_RESERVED, RIIC_ICBRL); - writeb(0, riic->base + RIIC_ICSER); - writeb(ICMR3_ACKWP | ICMR3_RDRFS, riic->base + RIIC_ICMR3); + riic_writeb(riic, 0, RIIC_ICSER); + riic_writeb(riic, ICMR3_ACKWP | ICMR3_RDRFS, RIIC_ICMR3); riic_clear_set_bit(riic, ICCR1_IICRST, 0, RIIC_ICCR1); @@ -443,6 +461,8 @@ static int riic_i2c_probe(struct platform_device *pdev) } } + riic->info = of_device_get_match_data(&pdev->dev); + adap = &riic->adapter; i2c_set_adapdata(adap, riic); strscpy(adap->name, "Renesas RIIC adapter", sizeof(adap->name)); @@ -481,14 +501,47 @@ static void riic_i2c_remove(struct platform_device *pdev) struct riic_dev *riic = platform_get_drvdata(pdev); pm_runtime_get_sync(&pdev->dev); - writeb(0, riic->base + RIIC_ICIER); + riic_writeb(riic, 0, RIIC_ICIER); pm_runtime_put(&pdev->dev); i2c_del_adapter(&riic->adapter); pm_runtime_disable(&pdev->dev); } +static const struct riic_of_data riic_rz_a_info = { + .regs = { + [RIIC_ICCR1] = 0x00, + [RIIC_ICCR2] = 0x04, + [RIIC_ICMR1] = 0x08, + [RIIC_ICMR3] = 0x10, + [RIIC_ICSER] = 0x18, + [RIIC_ICIER] = 0x1c, + [RIIC_ICSR2] = 0x24, + [RIIC_ICBRL] = 0x34, + [RIIC_ICBRH] = 0x38, + [RIIC_ICDRT] = 0x3c, + [RIIC_ICDRR] = 0x40, + }, +}; + +static const struct riic_of_data riic_rz_v2h_info = { + .regs = { + [RIIC_ICCR1] = 0x00, + [RIIC_ICCR2] = 0x01, + [RIIC_ICMR1] = 0x02, + [RIIC_ICMR3] = 0x04, + [RIIC_ICSER] = 0x06, + [RIIC_ICIER] = 0x07, + [RIIC_ICSR2] = 0x09, + [RIIC_ICBRL] = 0x10, + [RIIC_ICBRH] = 0x11, + [RIIC_ICDRT] = 0x12, + [RIIC_ICDRR] = 0x13, + }, +}; + static const struct of_device_id riic_i2c_dt_ids[] = { - { .compatible = "renesas,riic-rz", }, + { .compatible = "renesas,riic-rz", .data = &riic_rz_a_info }, + { .compatible = "renesas,riic-r9a09g057", .data = &riic_rz_v2h_info }, { /* Sentinel */ }, }; diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c index 086fdf262e7b..beca61700c89 100644 --- a/drivers/i2c/busses/i2c-rk3x.c +++ b/drivers/i2c/busses/i2c-rk3x.c @@ -1060,7 +1060,8 @@ static int rk3x_i2c_xfer_common(struct i2c_adapter *adap, struct i2c_msg *msgs, int num, bool polling) { struct rk3x_i2c *i2c = (struct rk3x_i2c *)adap->algo_data; - unsigned long timeout, flags; + unsigned long flags; + long time_left; u32 val; int ret = 0; int i; @@ -1092,23 +1093,20 @@ static int rk3x_i2c_xfer_common(struct i2c_adapter *adap, if (!polling) { rk3x_i2c_start(i2c); - timeout = wait_event_timeout(i2c->wait, !i2c->busy, - msecs_to_jiffies(WAIT_TIMEOUT)); + time_left = wait_event_timeout(i2c->wait, !i2c->busy, + msecs_to_jiffies(WAIT_TIMEOUT)); } else { disable_irq(i2c->irq); rk3x_i2c_start(i2c); - timeout = rk3x_i2c_wait_xfer_poll(i2c); + time_left = rk3x_i2c_wait_xfer_poll(i2c); enable_irq(i2c->irq); } spin_lock_irqsave(&i2c->lock, flags); - if (timeout == 0) { - dev_err(i2c->dev, "timeout, ipd: 0x%02x, state: %d\n", - i2c_readl(i2c, REG_IPD), i2c->state); - + if (time_left == 0) { /* Force a STOP condition without interrupt */ i2c_writel(i2c, 0, REG_IEN); val = i2c_readl(i2c, REG_CON) & REG_CON_TUNING_MASK; diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c index 275f7c42165c..01419c738cfc 100644 --- a/drivers/i2c/busses/i2c-s3c2410.c +++ b/drivers/i2c/busses/i2c-s3c2410.c @@ -685,7 +685,7 @@ static void s3c24xx_i2c_wait_idle(struct s3c24xx_i2c *i2c) static int s3c24xx_i2c_doxfer(struct s3c24xx_i2c *i2c, struct i2c_msg *msgs, int num) { - unsigned long timeout = 0; + long time_left = 0; int ret; ret = s3c24xx_i2c_set_master(i2c); @@ -715,7 +715,7 @@ static int s3c24xx_i2c_doxfer(struct s3c24xx_i2c *i2c, dev_err(i2c->dev, "deal with arbitration loss\n"); } } else { - timeout = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); + time_left = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); } ret = i2c->msg_idx; @@ -724,7 +724,7 @@ static int s3c24xx_i2c_doxfer(struct s3c24xx_i2c *i2c, * Having these next two as dev_err() makes life very * noisy when doing an i2cdetect */ - if (timeout == 0) + if (time_left == 0) dev_dbg(i2c->dev, "timeout\n"); else if (ret != num) dev_dbg(i2c->dev, "incomplete xfer (%d)\n", ret); diff --git a/drivers/i2c/busses/i2c-sh_mobile.c b/drivers/i2c/busses/i2c-sh_mobile.c index c65ac3d7eadc..f86c29737df1 100644 --- a/drivers/i2c/busses/i2c-sh_mobile.c +++ b/drivers/i2c/busses/i2c-sh_mobile.c @@ -688,7 +688,6 @@ static int sh_mobile_xfer(struct sh_mobile_i2c_data *pd, } if (!time_left) { - dev_err(pd->dev, "Transfer request timed out\n"); if (pd->dma_direction != DMA_NONE) sh_mobile_i2c_cleanup_dma(pd, true); diff --git a/drivers/i2c/busses/i2c-st.c b/drivers/i2c/busses/i2c-st.c index ce2333408904..5e01fe3dbb63 100644 --- a/drivers/i2c/busses/i2c-st.c +++ b/drivers/i2c/busses/i2c-st.c @@ -647,7 +647,7 @@ static int st_i2c_xfer_msg(struct st_i2c_dev *i2c_dev, struct i2c_msg *msg, { struct st_i2c_client *c = &i2c_dev->client; u32 ctl, i2c, it; - unsigned long timeout; + unsigned long time_left; int ret; c->addr = i2c_8bit_addr_from_msg(msg); @@ -685,15 +685,12 @@ static int st_i2c_xfer_msg(struct st_i2c_dev *i2c_dev, struct i2c_msg *msg, st_i2c_set_bits(i2c_dev->base + SSC_I2C, SSC_I2C_STRTG); } - timeout = wait_for_completion_timeout(&i2c_dev->complete, - i2c_dev->adap.timeout); + time_left = wait_for_completion_timeout(&i2c_dev->complete, + i2c_dev->adap.timeout); ret = c->result; - if (!timeout) { - dev_err(i2c_dev->dev, "Write to slave 0x%x timed out\n", - c->addr); + if (!time_left) ret = -ETIMEDOUT; - } i2c = SSC_I2C_STOPG | SSC_I2C_REPSTRTG; st_i2c_clr_bits(i2c_dev->base + SSC_I2C, i2c); diff --git a/drivers/i2c/busses/i2c-stm32f4.c b/drivers/i2c/busses/i2c-stm32f4.c index 859ac0cf7f6c..f8b12be6ef55 100644 --- a/drivers/i2c/busses/i2c-stm32f4.c +++ b/drivers/i2c/busses/i2c-stm32f4.c @@ -681,7 +681,7 @@ static int stm32f4_i2c_xfer_msg(struct stm32f4_i2c_dev *i2c_dev, { struct stm32f4_i2c_msg *f4_msg = &i2c_dev->msg; void __iomem *reg = i2c_dev->base + STM32F4_I2C_CR1; - unsigned long timeout; + unsigned long time_left; u32 mask; int ret; @@ -706,11 +706,11 @@ static int stm32f4_i2c_xfer_msg(struct stm32f4_i2c_dev *i2c_dev, stm32f4_i2c_set_bits(reg, STM32F4_I2C_CR1_START); } - timeout = wait_for_completion_timeout(&i2c_dev->complete, - i2c_dev->adap.timeout); + time_left = wait_for_completion_timeout(&i2c_dev->complete, + i2c_dev->adap.timeout); ret = f4_msg->result; - if (!timeout) + if (!time_left) ret = -ETIMEDOUT; return ret; diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c index 01210452216b..cfee2d9c09de 100644 --- a/drivers/i2c/busses/i2c-stm32f7.c +++ b/drivers/i2c/busses/i2c-stm32f7.c @@ -1789,7 +1789,7 @@ static int stm32f7_i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr, struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg; struct stm32_i2c_dma *dma = i2c_dev->dma; struct device *dev = i2c_dev->dev; - unsigned long timeout; + unsigned long time_left; int i, ret; f7_msg->addr = addr; @@ -1809,8 +1809,8 @@ static int stm32f7_i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr, if (ret) goto pm_free; - timeout = wait_for_completion_timeout(&i2c_dev->complete, - i2c_dev->adap.timeout); + time_left = wait_for_completion_timeout(&i2c_dev->complete, + i2c_dev->adap.timeout); ret = f7_msg->result; if (ret) { if (i2c_dev->use_dma) @@ -1826,7 +1826,7 @@ static int stm32f7_i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr, goto pm_free; } - if (!timeout) { + if (!time_left) { dev_dbg(dev, "Access to slave 0x%x timed out\n", f7_msg->addr); if (i2c_dev->use_dma) dmaengine_terminate_sync(dma->chan_using); diff --git a/drivers/i2c/busses/i2c-synquacer.c b/drivers/i2c/busses/i2c-synquacer.c index bbea521b05dd..31ecb2c7e978 100644 --- a/drivers/i2c/busses/i2c-synquacer.c +++ b/drivers/i2c/busses/i2c-synquacer.c @@ -311,7 +311,7 @@ static int synquacer_i2c_doxfer(struct synquacer_i2c *i2c, struct i2c_msg *msgs, int num) { unsigned char bsr; - unsigned long timeout; + unsigned long time_left; int ret; synquacer_i2c_hw_init(i2c); @@ -335,9 +335,9 @@ static int synquacer_i2c_doxfer(struct synquacer_i2c *i2c, return ret; } - timeout = wait_for_completion_timeout(&i2c->completion, - msecs_to_jiffies(i2c->timeout_ms)); - if (timeout == 0) { + time_left = wait_for_completion_timeout(&i2c->completion, + msecs_to_jiffies(i2c->timeout_ms)); + if (time_left == 0) { dev_dbg(i2c->dev, "timeout\n"); return -EAGAIN; } @@ -550,17 +550,13 @@ static int synquacer_i2c_probe(struct platform_device *pdev) device_property_read_u32(&pdev->dev, "socionext,pclk-rate", &i2c->pclkrate); - i2c->pclk = devm_clk_get(&pdev->dev, "pclk"); - if (PTR_ERR(i2c->pclk) == -EPROBE_DEFER) - return -EPROBE_DEFER; - if (!IS_ERR_OR_NULL(i2c->pclk)) { - dev_dbg(&pdev->dev, "clock source %p\n", i2c->pclk); + i2c->pclk = devm_clk_get_enabled(&pdev->dev, "pclk"); + if (IS_ERR(i2c->pclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(i2c->pclk), + "failed to get and enable clock\n"); - ret = clk_prepare_enable(i2c->pclk); - if (ret) - return dev_err_probe(&pdev->dev, ret, "failed to enable clock\n"); - i2c->pclkrate = clk_get_rate(i2c->pclk); - } + dev_dbg(&pdev->dev, "clock source %p\n", i2c->pclk); + i2c->pclkrate = clk_get_rate(i2c->pclk); if (i2c->pclkrate < SYNQUACER_I2C_MIN_CLK_RATE || i2c->pclkrate > SYNQUACER_I2C_MAX_CLK_RATE) @@ -615,8 +611,6 @@ static void synquacer_i2c_remove(struct platform_device *pdev) struct synquacer_i2c *i2c = platform_get_drvdata(pdev); i2c_del_adapter(&i2c->adapter); - if (!IS_ERR(i2c->pclk)) - clk_disable_unprepare(i2c->pclk); }; static const struct of_device_id synquacer_i2c_dt_ids[] __maybe_unused = { diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index 920d5a8cbf4c..85b31edc558d 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -1331,7 +1331,6 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, dmaengine_terminate_sync(i2c_dev->dma_chan); if (!time_left && !completion_done(&i2c_dev->dma_complete)) { - dev_err(i2c_dev->dev, "DMA transfer timed out\n"); tegra_i2c_init(i2c_dev); return -ETIMEDOUT; } @@ -1351,7 +1350,6 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, tegra_i2c_mask_irq(i2c_dev, int_mask); if (time_left == 0) { - dev_err(i2c_dev->dev, "I2C transfer timed out\n"); tegra_i2c_init(i2c_dev); return -ETIMEDOUT; } diff --git a/drivers/i2c/busses/i2c-thunderx-pcidrv.c b/drivers/i2c/busses/i2c-thunderx-pcidrv.c index a77cd86fe75e..32d0e3930b67 100644 --- a/drivers/i2c/busses/i2c-thunderx-pcidrv.c +++ b/drivers/i2c/busses/i2c-thunderx-pcidrv.c @@ -27,7 +27,8 @@ #define PCI_DEVICE_ID_THUNDER_TWSI 0xa012 -#define SYS_FREQ_DEFAULT 700000000 +#define SYS_FREQ_DEFAULT 800000000 +#define OTX2_REF_FREQ_DEFAULT 100000000 #define TWSI_INT_ENA_W1C 0x1028 #define TWSI_INT_ENA_W1S 0x1030 @@ -99,7 +100,8 @@ static void thunder_i2c_clock_enable(struct device *dev, struct octeon_i2c *i2c) i2c->sys_freq = clk_get_rate(i2c->clk); } else { /* ACPI */ - device_property_read_u32(dev, "sclk", &i2c->sys_freq); + if (device_property_read_u32(dev, "sclk", &i2c->sys_freq)) + device_property_read_u32(dev, "ioclk", &i2c->sys_freq); } skip: @@ -165,6 +167,7 @@ static int thunder_i2c_probe_pci(struct pci_dev *pdev, i2c->roff.sw_twsi = 0x1000; i2c->roff.twsi_int = 0x1010; i2c->roff.sw_twsi_ext = 0x1018; + i2c->roff.mode = 0x1038; i2c->dev = dev; pci_set_drvdata(pdev, i2c); @@ -205,6 +208,12 @@ static int thunder_i2c_probe_pci(struct pci_dev *pdev, if (ret) goto error; + /* + * For OcteonTX2 chips, set reference frequency to 100MHz + * as refclk_src in TWSI_MODE register defaults to 100MHz. + */ + if (octeon_i2c_is_otx2(pdev) && IS_LS_FREQ(i2c->twsi_freq)) + i2c->sys_freq = OTX2_REF_FREQ_DEFAULT; octeon_i2c_set_clock(i2c); i2c->adap = thunderx_i2c_ops; diff --git a/drivers/i2c/busses/i2c-uniphier-f.c b/drivers/i2c/busses/i2c-uniphier-f.c index dbc91c7c3788..6c3dac2cf568 100644 --- a/drivers/i2c/busses/i2c-uniphier-f.c +++ b/drivers/i2c/busses/i2c-uniphier-f.c @@ -358,7 +358,6 @@ static int uniphier_fi2c_master_xfer_one(struct i2c_adapter *adap, spin_unlock_irqrestore(&priv->lock, flags); if (!time_left) { - dev_err(&adap->dev, "transaction timeout.\n"); uniphier_fi2c_recover(priv); return -ETIMEDOUT; } diff --git a/drivers/i2c/busses/i2c-uniphier.c b/drivers/i2c/busses/i2c-uniphier.c index 854ac25b5862..e1b4c80e0285 100644 --- a/drivers/i2c/busses/i2c-uniphier.c +++ b/drivers/i2c/busses/i2c-uniphier.c @@ -71,10 +71,8 @@ static int uniphier_i2c_xfer_byte(struct i2c_adapter *adap, u32 txdata, writel(txdata, priv->membase + UNIPHIER_I2C_DTRM); time_left = wait_for_completion_timeout(&priv->comp, adap->timeout); - if (unlikely(!time_left)) { - dev_err(&adap->dev, "transaction timeout\n"); + if (unlikely(!time_left)) return -ETIMEDOUT; - } rxdata = readl(priv->membase + UNIPHIER_I2C_DREC); if (rxdatap) diff --git a/drivers/i2c/busses/i2c-viai2c-common.c b/drivers/i2c/busses/i2c-viai2c-common.c new file mode 100644 index 000000000000..1844d13f1f79 --- /dev/null +++ b/drivers/i2c/busses/i2c-viai2c-common.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include <linux/of_irq.h> +#include "i2c-viai2c-common.h" + +int viai2c_wait_bus_not_busy(struct viai2c *i2c) +{ + unsigned long timeout; + + timeout = jiffies + VIAI2C_TIMEOUT; + while (!(readw(i2c->base + VIAI2C_REG_CSR) & VIAI2C_CSR_READY_MASK)) { + if (time_after(jiffies, timeout)) { + dev_warn(i2c->dev, "timeout waiting for bus ready\n"); + return -EBUSY; + } + msleep(20); + } + + return 0; +} + +static int viai2c_write(struct viai2c *i2c, struct i2c_msg *pmsg, int last) +{ + u16 val, tcr_val = i2c->tcr; + + i2c->last = last; + + if (pmsg->len == 0) { + /* + * We still need to run through the while (..) once, so + * start at -1 and break out early from the loop + */ + i2c->xfered_len = -1; + writew(0, i2c->base + VIAI2C_REG_CDR); + } else { + writew(pmsg->buf[0] & 0xFF, i2c->base + VIAI2C_REG_CDR); + } + + if (i2c->platform == VIAI2C_PLAT_WMT && !(pmsg->flags & I2C_M_NOSTART)) { + val = readw(i2c->base + VIAI2C_REG_CR); + val &= ~VIAI2C_CR_TX_END; + val |= VIAI2C_CR_CPU_RDY; + writew(val, i2c->base + VIAI2C_REG_CR); + } + + reinit_completion(&i2c->complete); + + tcr_val |= pmsg->addr & VIAI2C_TCR_ADDR_MASK; + + writew(tcr_val, i2c->base + VIAI2C_REG_TCR); + + if (i2c->platform == VIAI2C_PLAT_WMT && pmsg->flags & I2C_M_NOSTART) { + val = readw(i2c->base + VIAI2C_REG_CR); + val |= VIAI2C_CR_CPU_RDY; + writew(val, i2c->base + VIAI2C_REG_CR); + } + + if (!wait_for_completion_timeout(&i2c->complete, VIAI2C_TIMEOUT)) + return -ETIMEDOUT; + + return i2c->ret; +} + +static int viai2c_read(struct viai2c *i2c, struct i2c_msg *pmsg, bool first) +{ + u16 val, tcr_val = i2c->tcr; + + val = readw(i2c->base + VIAI2C_REG_CR); + val &= ~(VIAI2C_CR_TX_END | VIAI2C_CR_RX_END); + + if (i2c->platform == VIAI2C_PLAT_WMT && !(pmsg->flags & I2C_M_NOSTART)) + val |= VIAI2C_CR_CPU_RDY; + + if (pmsg->len == 1) + val |= VIAI2C_CR_RX_END; + + writew(val, i2c->base + VIAI2C_REG_CR); + + reinit_completion(&i2c->complete); + + tcr_val |= VIAI2C_TCR_READ | (pmsg->addr & VIAI2C_TCR_ADDR_MASK); + + writew(tcr_val, i2c->base + VIAI2C_REG_TCR); + + if ((i2c->platform == VIAI2C_PLAT_WMT && (pmsg->flags & I2C_M_NOSTART)) || + (i2c->platform == VIAI2C_PLAT_ZHAOXIN && !first)) { + val = readw(i2c->base + VIAI2C_REG_CR); + val |= VIAI2C_CR_CPU_RDY; + writew(val, i2c->base + VIAI2C_REG_CR); + } + + if (!wait_for_completion_timeout(&i2c->complete, VIAI2C_TIMEOUT)) + return -ETIMEDOUT; + + return i2c->ret; +} + +int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) +{ + struct i2c_msg *pmsg; + int i; + int ret = 0; + struct viai2c *i2c = i2c_get_adapdata(adap); + + i2c->mode = VIAI2C_BYTE_MODE; + for (i = 0; ret >= 0 && i < num; i++) { + pmsg = &msgs[i]; + if (i2c->platform == VIAI2C_PLAT_WMT && !(pmsg->flags & I2C_M_NOSTART)) { + ret = viai2c_wait_bus_not_busy(i2c); + if (ret < 0) + return ret; + } + + i2c->msg = pmsg; + i2c->xfered_len = 0; + + if (pmsg->flags & I2C_M_RD) + ret = viai2c_read(i2c, pmsg, i == 0); + else + ret = viai2c_write(i2c, pmsg, (i + 1) == num); + } + + return (ret < 0) ? ret : i; +} + +/* + * Main process of the byte mode xfer + * + * Return value indicates whether the transfer is complete + * 1: all the data has been successfully transferred + * 0: there is still data that needs to be transferred + * -EIO: error occurred + */ +static int viai2c_irq_xfer(struct viai2c *i2c) +{ + u16 val; + struct i2c_msg *msg = i2c->msg; + u8 read = msg->flags & I2C_M_RD; + void __iomem *base = i2c->base; + + if (read) { + msg->buf[i2c->xfered_len] = readw(base + VIAI2C_REG_CDR) >> 8; + + val = readw(base + VIAI2C_REG_CR) | VIAI2C_CR_CPU_RDY; + if (i2c->xfered_len == msg->len - 2) + val |= VIAI2C_CR_RX_END; + writew(val, base + VIAI2C_REG_CR); + } else { + val = readw(base + VIAI2C_REG_CSR); + if (val & VIAI2C_CSR_RCV_NOT_ACK) + return -EIO; + + /* I2C_SMBUS_QUICK */ + if (msg->len == 0) { + val = VIAI2C_CR_TX_END | VIAI2C_CR_CPU_RDY | VIAI2C_CR_ENABLE; + writew(val, base + VIAI2C_REG_CR); + return 1; + } + + if ((i2c->xfered_len + 1) == msg->len) { + if (i2c->platform == VIAI2C_PLAT_WMT && !i2c->last) + writew(VIAI2C_CR_ENABLE, base + VIAI2C_REG_CR); + else if (i2c->platform == VIAI2C_PLAT_ZHAOXIN && i2c->last) + writeb(VIAI2C_CR_TX_END, base + VIAI2C_REG_CR); + } else { + writew(msg->buf[i2c->xfered_len + 1] & 0xFF, base + VIAI2C_REG_CDR); + writew(VIAI2C_CR_CPU_RDY | VIAI2C_CR_ENABLE, base + VIAI2C_REG_CR); + } + } + + i2c->xfered_len++; + + return i2c->xfered_len == msg->len; +} + +int __weak viai2c_fifo_irq_xfer(struct viai2c *i2c, bool irq) +{ + return 0; +} + +static irqreturn_t viai2c_isr(int irq, void *data) +{ + struct viai2c *i2c = data; + u8 status; + + /* save the status and write-clear it */ + status = readw(i2c->base + VIAI2C_REG_ISR); + if (!status && i2c->platform == VIAI2C_PLAT_ZHAOXIN) + return IRQ_NONE; + + writew(status, i2c->base + VIAI2C_REG_ISR); + + i2c->ret = 0; + if (status & VIAI2C_ISR_NACK_ADDR) + i2c->ret = -EIO; + + if (i2c->platform == VIAI2C_PLAT_WMT && (status & VIAI2C_ISR_SCL_TIMEOUT)) + i2c->ret = -ETIMEDOUT; + + if (!i2c->ret) { + if (i2c->mode == VIAI2C_BYTE_MODE) + i2c->ret = viai2c_irq_xfer(i2c); + else + i2c->ret = viai2c_fifo_irq_xfer(i2c, true); + } + + /* All the data has been successfully transferred or error occurred */ + if (i2c->ret) + complete(&i2c->complete); + + return IRQ_HANDLED; +} + +int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c, int plat) +{ + int err; + int irq_flags; + struct viai2c *i2c; + struct device_node *np = pdev->dev.of_node; + + i2c = devm_kzalloc(&pdev->dev, sizeof(*i2c), GFP_KERNEL); + if (!i2c) + return -ENOMEM; + + i2c->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(i2c->base)) + return PTR_ERR(i2c->base); + + if (plat == VIAI2C_PLAT_WMT) { + irq_flags = 0; + i2c->irq = irq_of_parse_and_map(np, 0); + if (!i2c->irq) + return -EINVAL; + } else if (plat == VIAI2C_PLAT_ZHAOXIN) { + irq_flags = IRQF_SHARED; + i2c->irq = platform_get_irq(pdev, 0); + if (i2c->irq < 0) + return i2c->irq; + } else { + return dev_err_probe(&pdev->dev, -EINVAL, "wrong platform type\n"); + } + + i2c->platform = plat; + + err = devm_request_irq(&pdev->dev, i2c->irq, viai2c_isr, + irq_flags, pdev->name, i2c); + if (err) + return dev_err_probe(&pdev->dev, err, + "failed to request irq %i\n", i2c->irq); + + i2c->dev = &pdev->dev; + init_completion(&i2c->complete); + platform_set_drvdata(pdev, i2c); + + *pi2c = i2c; + return 0; +} diff --git a/drivers/i2c/busses/i2c-viai2c-common.h b/drivers/i2c/busses/i2c-viai2c-common.h new file mode 100644 index 000000000000..81e827c54434 --- /dev/null +++ b/drivers/i2c/busses/i2c-viai2c-common.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __I2C_VIAI2C_COMMON_H_ +#define __I2C_VIAI2C_COMMON_H_ + +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/i2c.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> + +/* REG_CR Bit fields */ +#define VIAI2C_REG_CR 0x00 +#define VIAI2C_CR_ENABLE BIT(0) +#define VIAI2C_CR_RX_END BIT(1) +#define VIAI2C_CR_TX_END BIT(2) +#define VIAI2C_CR_CPU_RDY BIT(3) +#define VIAI2C_CR_END_MASK GENMASK(2, 1) + +/* REG_TCR Bit fields */ +#define VIAI2C_REG_TCR 0x02 +#define VIAI2C_TCR_HS_MODE BIT(13) +#define VIAI2C_TCR_READ BIT(14) +#define VIAI2C_TCR_FAST BIT(15) +#define VIAI2C_TCR_ADDR_MASK GENMASK(6, 0) + +/* REG_CSR Bit fields */ +#define VIAI2C_REG_CSR 0x04 +#define VIAI2C_CSR_RCV_NOT_ACK BIT(0) +#define VIAI2C_CSR_RCV_ACK_MASK BIT(0) +#define VIAI2C_CSR_READY_MASK BIT(1) + +/* REG_ISR Bit fields */ +#define VIAI2C_REG_ISR 0x06 +#define VIAI2C_ISR_NACK_ADDR BIT(0) +#define VIAI2C_ISR_BYTE_END BIT(1) +#define VIAI2C_ISR_SCL_TIMEOUT BIT(2) +#define VIAI2C_ISR_MASK_ALL GENMASK(2, 0) + +/* REG_IMR Bit fields */ +#define VIAI2C_REG_IMR 0x08 +#define VIAI2C_IMR_BYTE BIT(1) +#define VIAI2C_IMR_ENABLE_ALL GENMASK(2, 0) + +#define VIAI2C_REG_CDR 0x0A +#define VIAI2C_REG_TR 0x0C +#define VIAI2C_REG_MCR 0x0E + +#define VIAI2C_TIMEOUT (msecs_to_jiffies(1000)) + +enum { + VIAI2C_PLAT_WMT, + VIAI2C_PLAT_ZHAOXIN +}; + +enum { + VIAI2C_BYTE_MODE, + VIAI2C_FIFO_MODE +}; + +struct viai2c { + struct i2c_adapter adapter; + struct completion complete; + struct device *dev; + void __iomem *base; + struct clk *clk; + u16 tcr; + int irq; + u16 xfered_len; + struct i2c_msg *msg; + int ret; + bool last; + unsigned int mode; + unsigned int platform; + void *pltfm_priv; +}; + +int viai2c_wait_bus_not_busy(struct viai2c *i2c); +int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num); +int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c, int plat); +int viai2c_fifo_irq_xfer(struct viai2c *i2c, bool irq); + +#endif diff --git a/drivers/i2c/busses/i2c-viai2c-wmt.c b/drivers/i2c/busses/i2c-viai2c-wmt.c new file mode 100644 index 000000000000..e1988f946026 --- /dev/null +++ b/drivers/i2c/busses/i2c-viai2c-wmt.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Wondermedia I2C Master Mode Driver + * + * Copyright (C) 2012 Tony Prisk <linux@prisktech.co.nz> + * + * Derived from GPLv2+ licensed source: + * - Copyright (C) 2008 WonderMedia Technologies, Inc. + */ + +#include <linux/clk.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include "i2c-viai2c-common.h" + +#define REG_SLAVE_CR 0x10 +#define REG_SLAVE_SR 0x12 +#define REG_SLAVE_ISR 0x14 +#define REG_SLAVE_IMR 0x16 +#define REG_SLAVE_DR 0x18 +#define REG_SLAVE_TR 0x1A + +/* REG_TR */ +#define SCL_TIMEOUT(x) (((x) & 0xFF) << 8) +#define TR_STD 0x0064 +#define TR_HS 0x0019 + +/* REG_MCR */ +#define MCR_APB_96M 7 +#define MCR_APB_166M 12 + +static u32 wmt_i2c_func(struct i2c_adapter *adap) +{ + return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | I2C_FUNC_NOSTART; +} + +static const struct i2c_algorithm wmt_i2c_algo = { + .master_xfer = viai2c_xfer, + .functionality = wmt_i2c_func, +}; + +static int wmt_i2c_reset_hardware(struct viai2c *i2c) +{ + int err; + + err = clk_prepare_enable(i2c->clk); + if (err) { + dev_err(i2c->dev, "failed to enable clock\n"); + return err; + } + + err = clk_set_rate(i2c->clk, 20000000); + if (err) { + dev_err(i2c->dev, "failed to set clock = 20Mhz\n"); + clk_disable_unprepare(i2c->clk); + return err; + } + + writew(0, i2c->base + VIAI2C_REG_CR); + writew(MCR_APB_166M, i2c->base + VIAI2C_REG_MCR); + writew(VIAI2C_ISR_MASK_ALL, i2c->base + VIAI2C_REG_ISR); + writew(VIAI2C_IMR_ENABLE_ALL, i2c->base + VIAI2C_REG_IMR); + writew(VIAI2C_CR_ENABLE, i2c->base + VIAI2C_REG_CR); + readw(i2c->base + VIAI2C_REG_CSR); /* read clear */ + writew(VIAI2C_ISR_MASK_ALL, i2c->base + VIAI2C_REG_ISR); + + if (i2c->tcr == VIAI2C_TCR_FAST) + writew(SCL_TIMEOUT(128) | TR_HS, i2c->base + VIAI2C_REG_TR); + else + writew(SCL_TIMEOUT(128) | TR_STD, i2c->base + VIAI2C_REG_TR); + + return 0; +} + +static int wmt_i2c_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct viai2c *i2c; + struct i2c_adapter *adap; + int err; + u32 clk_rate; + + err = viai2c_init(pdev, &i2c, VIAI2C_PLAT_WMT); + if (err) + return err; + + i2c->clk = of_clk_get(np, 0); + if (IS_ERR(i2c->clk)) { + dev_err(&pdev->dev, "unable to request clock\n"); + return PTR_ERR(i2c->clk); + } + + err = of_property_read_u32(np, "clock-frequency", &clk_rate); + if (!err && clk_rate == I2C_MAX_FAST_MODE_FREQ) + i2c->tcr = VIAI2C_TCR_FAST; + + adap = &i2c->adapter; + i2c_set_adapdata(adap, i2c); + strscpy(adap->name, "WMT I2C adapter", sizeof(adap->name)); + adap->owner = THIS_MODULE; + adap->algo = &wmt_i2c_algo; + adap->dev.parent = &pdev->dev; + adap->dev.of_node = pdev->dev.of_node; + + err = wmt_i2c_reset_hardware(i2c); + if (err) { + dev_err(&pdev->dev, "error initializing hardware\n"); + return err; + } + + err = i2c_add_adapter(adap); + if (err) + /* wmt_i2c_reset_hardware() enables i2c_dev->clk */ + clk_disable_unprepare(i2c->clk); + + return err; +} + +static void wmt_i2c_remove(struct platform_device *pdev) +{ + struct viai2c *i2c = platform_get_drvdata(pdev); + + /* Disable interrupts, clock and delete adapter */ + writew(0, i2c->base + VIAI2C_REG_IMR); + clk_disable_unprepare(i2c->clk); + i2c_del_adapter(&i2c->adapter); +} + +static const struct of_device_id wmt_i2c_dt_ids[] = { + { .compatible = "wm,wm8505-i2c" }, + { /* Sentinel */ }, +}; + +static struct platform_driver wmt_i2c_driver = { + .probe = wmt_i2c_probe, + .remove_new = wmt_i2c_remove, + .driver = { + .name = "wmt-i2c", + .of_match_table = wmt_i2c_dt_ids, + }, +}; + +module_platform_driver(wmt_i2c_driver); + +MODULE_DESCRIPTION("Wondermedia I2C master-mode bus adapter"); +MODULE_AUTHOR("Tony Prisk <linux@prisktech.co.nz>"); +MODULE_LICENSE("GPL"); +MODULE_DEVICE_TABLE(of, wmt_i2c_dt_ids); diff --git a/drivers/i2c/busses/i2c-viai2c-zhaoxin.c b/drivers/i2c/busses/i2c-viai2c-zhaoxin.c new file mode 100644 index 000000000000..7e3ac2a3e1fd --- /dev/null +++ b/drivers/i2c/busses/i2c-viai2c-zhaoxin.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright(c) 2024 Shanghai Zhaoxin Semiconductor Corporation. + * All rights reserved. + */ + +#include <linux/acpi.h> +#include "i2c-viai2c-common.h" + +/* + * registers + */ +/* Zhaoxin specific register bit fields */ +/* REG_CR Bit fields */ +#define ZXI2C_CR_MST_RST BIT(7) +#define ZXI2C_CR_FIFO_MODE BIT(14) +/* REG_ISR/IMR Bit fields */ +#define ZXI2C_IRQ_FIFONACK BIT(4) +#define ZXI2C_IRQ_FIFOEND BIT(3) +#define ZXI2C_IRQ_MASK (VIAI2C_ISR_MASK_ALL \ + | ZXI2C_IRQ_FIFOEND \ + | ZXI2C_IRQ_FIFONACK) +/* Zhaoxin specific registers */ +#define ZXI2C_REG_CLK 0x10 +#define ZXI2C_CLK_50M BIT(0) +#define ZXI2C_REG_REV 0x11 +#define ZXI2C_REG_HCR 0x12 +#define ZXI2C_HCR_RST_FIFO GENMASK(1, 0) +#define ZXI2C_REG_HTDR 0x13 +#define ZXI2C_REG_HRDR 0x14 +#define ZXI2C_REG_HTLR 0x15 +#define ZXI2C_REG_HRLR 0x16 +#define ZXI2C_REG_HWCNTR 0x18 +#define ZXI2C_REG_HRCNTR 0x19 + +/* parameters Constants */ +#define ZXI2C_GOLD_FSTP_100K 0xF3 +#define ZXI2C_GOLD_FSTP_400K 0x38 +#define ZXI2C_GOLD_FSTP_1M 0x13 +#define ZXI2C_GOLD_FSTP_3400K 0x37 +#define ZXI2C_HS_MASTER_CODE (0x08 << 8) + +#define ZXI2C_FIFO_SIZE 32 + +struct viai2c_zhaoxin { + u8 hrv; + u16 tr; + u16 mcr; + u16 xfer_len; +}; + +/* 'irq == true' means in interrupt context */ +int viai2c_fifo_irq_xfer(struct viai2c *i2c, bool irq) +{ + u16 i; + u8 tmp; + struct i2c_msg *msg = i2c->msg; + void __iomem *base = i2c->base; + bool read = !!(msg->flags & I2C_M_RD); + struct viai2c_zhaoxin *priv = i2c->pltfm_priv; + + if (irq) { + /* get the received data */ + if (read) + for (i = 0; i < priv->xfer_len; i++) + msg->buf[i2c->xfered_len + i] = ioread8(base + ZXI2C_REG_HRDR); + + i2c->xfered_len += priv->xfer_len; + if (i2c->xfered_len == msg->len) + return 1; + } + + /* reset fifo buffer */ + tmp = ioread8(base + ZXI2C_REG_HCR); + iowrite8(tmp | ZXI2C_HCR_RST_FIFO, base + ZXI2C_REG_HCR); + + /* set xfer len */ + priv->xfer_len = min_t(u16, msg->len - i2c->xfered_len, ZXI2C_FIFO_SIZE); + if (read) { + iowrite8(priv->xfer_len - 1, base + ZXI2C_REG_HRLR); + } else { + iowrite8(priv->xfer_len - 1, base + ZXI2C_REG_HTLR); + /* set write data */ + for (i = 0; i < priv->xfer_len; i++) + iowrite8(msg->buf[i2c->xfered_len + i], base + ZXI2C_REG_HTDR); + } + + /* prepare to stop transmission */ + if (priv->hrv && msg->len == (i2c->xfered_len + priv->xfer_len)) { + tmp = ioread8(base + VIAI2C_REG_CR); + tmp |= read ? VIAI2C_CR_RX_END : VIAI2C_CR_TX_END; + iowrite8(tmp, base + VIAI2C_REG_CR); + } + + if (irq) { + /* continue transmission */ + tmp = ioread8(base + VIAI2C_REG_CR); + iowrite8(tmp |= VIAI2C_CR_CPU_RDY, base + VIAI2C_REG_CR); + } else { + u16 tcr_val = i2c->tcr; + + /* start transmission */ + tcr_val |= read ? VIAI2C_TCR_READ : 0; + writew(tcr_val | msg->addr, base + VIAI2C_REG_TCR); + } + + return 0; +} + +static int zxi2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num) +{ + u8 tmp; + int ret; + struct viai2c *i2c = (struct viai2c *)i2c_get_adapdata(adap); + struct viai2c_zhaoxin *priv = i2c->pltfm_priv; + void __iomem *base = i2c->base; + + ret = viai2c_wait_bus_not_busy(i2c); + if (ret) + return ret; + + tmp = ioread8(base + VIAI2C_REG_CR); + tmp &= ~(VIAI2C_CR_RX_END | VIAI2C_CR_TX_END); + + if (num == 1 && msgs->len >= 2 && (priv->hrv || msgs->len <= ZXI2C_FIFO_SIZE)) { + /* enable fifo mode */ + iowrite16(ZXI2C_CR_FIFO_MODE | tmp, base + VIAI2C_REG_CR); + /* clear irq status */ + iowrite8(ZXI2C_IRQ_MASK, base + VIAI2C_REG_ISR); + /* enable fifo irq */ + iowrite8(VIAI2C_ISR_NACK_ADDR | ZXI2C_IRQ_FIFOEND, base + VIAI2C_REG_IMR); + + i2c->msg = msgs; + i2c->mode = VIAI2C_FIFO_MODE; + priv->xfer_len = 0; + i2c->xfered_len = 0; + + viai2c_fifo_irq_xfer(i2c, 0); + + if (!wait_for_completion_timeout(&i2c->complete, VIAI2C_TIMEOUT)) + return -ETIMEDOUT; + + ret = i2c->ret; + } else { + /* enable byte mode */ + iowrite16(tmp, base + VIAI2C_REG_CR); + /* clear irq status */ + iowrite8(ZXI2C_IRQ_MASK, base + VIAI2C_REG_ISR); + /* enable byte irq */ + iowrite8(VIAI2C_ISR_NACK_ADDR | VIAI2C_IMR_BYTE, base + VIAI2C_REG_IMR); + + ret = viai2c_xfer(adap, msgs, num); + if (ret == -ETIMEDOUT) + iowrite16(tmp | VIAI2C_CR_END_MASK, base + VIAI2C_REG_CR); + } + /* dis interrupt */ + iowrite8(0, base + VIAI2C_REG_IMR); + + return ret; +} + +static u32 zxi2c_func(struct i2c_adapter *adap) +{ + return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; +} + +static const struct i2c_algorithm zxi2c_algorithm = { + .master_xfer = zxi2c_master_xfer, + .functionality = zxi2c_func, +}; + +static const struct i2c_adapter_quirks zxi2c_quirks = { + .flags = I2C_AQ_NO_ZERO_LEN | I2C_AQ_COMB_WRITE_THEN_READ, +}; + +static const u32 zxi2c_speed_params_table[][3] = { + /* speed, ZXI2C_TCR, ZXI2C_FSTP */ + { I2C_MAX_STANDARD_MODE_FREQ, 0, ZXI2C_GOLD_FSTP_100K }, + { I2C_MAX_FAST_MODE_FREQ, VIAI2C_TCR_FAST, ZXI2C_GOLD_FSTP_400K }, + { I2C_MAX_FAST_MODE_PLUS_FREQ, VIAI2C_TCR_FAST, ZXI2C_GOLD_FSTP_1M }, + { I2C_MAX_HIGH_SPEED_MODE_FREQ, VIAI2C_TCR_HS_MODE | VIAI2C_TCR_FAST, + ZXI2C_GOLD_FSTP_3400K }, +}; + +static void zxi2c_set_bus_speed(struct viai2c *i2c) +{ + struct viai2c_zhaoxin *priv = i2c->pltfm_priv; + + iowrite16(priv->tr, i2c->base + VIAI2C_REG_TR); + iowrite8(ZXI2C_CLK_50M, i2c->base + ZXI2C_REG_CLK); + iowrite16(priv->mcr, i2c->base + VIAI2C_REG_MCR); +} + +static void zxi2c_get_bus_speed(struct viai2c *i2c) +{ + u8 i, count; + u8 fstp; + const u32 *params; + struct viai2c_zhaoxin *priv = i2c->pltfm_priv; + u32 acpi_speed = i2c_acpi_find_bus_speed(i2c->dev); + + count = ARRAY_SIZE(zxi2c_speed_params_table); + for (i = 0; i < count; i++) + if (acpi_speed == zxi2c_speed_params_table[i][0]) + break; + /* if not found, use 400k as default */ + i = i < count ? i : 1; + + params = zxi2c_speed_params_table[i]; + fstp = ioread8(i2c->base + VIAI2C_REG_TR); + if (abs(fstp - params[2]) > 0x10) { + /* + * if BIOS setting value far from golden value, + * use golden value and warn user + */ + dev_warn(i2c->dev, "FW FSTP[%x] might cause wrong timings, dropped\n", fstp); + priv->tr = params[2] | 0xff00; + } else { + priv->tr = fstp | 0xff00; + } + + i2c->tcr = params[1]; + priv->mcr = ioread16(i2c->base + VIAI2C_REG_MCR); + /* for Hs-mode, use 0x80 as master code */ + if (params[0] == I2C_MAX_HIGH_SPEED_MODE_FREQ) + priv->mcr |= ZXI2C_HS_MASTER_CODE; + + dev_info(i2c->dev, "speed mode is %s\n", i2c_freq_mode_string(params[0])); +} + +static int zxi2c_probe(struct platform_device *pdev) +{ + int error; + struct viai2c *i2c; + struct i2c_adapter *adap; + struct viai2c_zhaoxin *priv; + + error = viai2c_init(pdev, &i2c, VIAI2C_PLAT_ZHAOXIN); + if (error) + return error; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + i2c->pltfm_priv = priv; + + zxi2c_get_bus_speed(i2c); + zxi2c_set_bus_speed(i2c); + + priv->hrv = ioread8(i2c->base + ZXI2C_REG_REV); + + adap = &i2c->adapter; + adap->owner = THIS_MODULE; + adap->algo = &zxi2c_algorithm; + adap->quirks = &zxi2c_quirks; + adap->dev.parent = &pdev->dev; + ACPI_COMPANION_SET(&adap->dev, ACPI_COMPANION(&pdev->dev)); + snprintf(adap->name, sizeof(adap->name), "zhaoxin-%s-%s", + dev_name(pdev->dev.parent), dev_name(i2c->dev)); + i2c_set_adapdata(adap, i2c); + + return devm_i2c_add_adapter(&pdev->dev, adap); +} + +static int __maybe_unused zxi2c_resume(struct device *dev) +{ + struct viai2c *i2c = dev_get_drvdata(dev); + + iowrite8(ZXI2C_CR_MST_RST, i2c->base + VIAI2C_REG_CR); + zxi2c_set_bus_speed(i2c); + + return 0; +} + +static const struct dev_pm_ops zxi2c_pm = { + SET_SYSTEM_SLEEP_PM_OPS(NULL, zxi2c_resume) +}; + +static const struct acpi_device_id zxi2c_acpi_match[] = { + {"IIC1D17", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, zxi2c_acpi_match); + +static struct platform_driver zxi2c_driver = { + .probe = zxi2c_probe, + .driver = { + .name = "i2c_zhaoxin", + .acpi_match_table = zxi2c_acpi_match, + .pm = &zxi2c_pm, + }, +}; + +module_platform_driver(zxi2c_driver); + +MODULE_AUTHOR("HansHu@zhaoxin.com"); +MODULE_DESCRIPTION("Shanghai Zhaoxin IIC driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-viperboard.c b/drivers/i2c/busses/i2c-viperboard.c index 9e153b5b0e8e..3784b07f5371 100644 --- a/drivers/i2c/busses/i2c-viperboard.c +++ b/drivers/i2c/busses/i2c-viperboard.c @@ -416,7 +416,6 @@ static void vprbrd_i2c_remove(struct platform_device *pdev) static struct platform_driver vprbrd_i2c_driver = { .driver.name = "viperboard-i2c", - .driver.owner = THIS_MODULE, .probe = vprbrd_i2c_probe, .remove_new = vprbrd_i2c_remove, }; diff --git a/drivers/i2c/busses/i2c-wmt.c b/drivers/i2c/busses/i2c-wmt.c deleted file mode 100644 index 198afee5233c..000000000000 --- a/drivers/i2c/busses/i2c-wmt.c +++ /dev/null @@ -1,421 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Wondermedia I2C Master Mode Driver - * - * Copyright (C) 2012 Tony Prisk <linux@prisktech.co.nz> - * - * Derived from GPLv2+ licensed source: - * - Copyright (C) 2008 WonderMedia Technologies, Inc. - */ - -#include <linux/clk.h> -#include <linux/delay.h> -#include <linux/err.h> -#include <linux/i2c.h> -#include <linux/interrupt.h> -#include <linux/io.h> -#include <linux/module.h> -#include <linux/of.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/platform_device.h> - -#define REG_CR 0x00 -#define REG_TCR 0x02 -#define REG_CSR 0x04 -#define REG_ISR 0x06 -#define REG_IMR 0x08 -#define REG_CDR 0x0A -#define REG_TR 0x0C -#define REG_MCR 0x0E -#define REG_SLAVE_CR 0x10 -#define REG_SLAVE_SR 0x12 -#define REG_SLAVE_ISR 0x14 -#define REG_SLAVE_IMR 0x16 -#define REG_SLAVE_DR 0x18 -#define REG_SLAVE_TR 0x1A - -/* REG_CR Bit fields */ -#define CR_TX_NEXT_ACK 0x0000 -#define CR_ENABLE 0x0001 -#define CR_TX_NEXT_NO_ACK 0x0002 -#define CR_TX_END 0x0004 -#define CR_CPU_RDY 0x0008 -#define SLAV_MODE_SEL 0x8000 - -/* REG_TCR Bit fields */ -#define TCR_STANDARD_MODE 0x0000 -#define TCR_MASTER_WRITE 0x0000 -#define TCR_HS_MODE 0x2000 -#define TCR_MASTER_READ 0x4000 -#define TCR_FAST_MODE 0x8000 -#define TCR_SLAVE_ADDR_MASK 0x007F - -/* REG_ISR Bit fields */ -#define ISR_NACK_ADDR 0x0001 -#define ISR_BYTE_END 0x0002 -#define ISR_SCL_TIMEOUT 0x0004 -#define ISR_WRITE_ALL 0x0007 - -/* REG_IMR Bit fields */ -#define IMR_ENABLE_ALL 0x0007 - -/* REG_CSR Bit fields */ -#define CSR_RCV_NOT_ACK 0x0001 -#define CSR_RCV_ACK_MASK 0x0001 -#define CSR_READY_MASK 0x0002 - -/* REG_TR */ -#define SCL_TIMEOUT(x) (((x) & 0xFF) << 8) -#define TR_STD 0x0064 -#define TR_HS 0x0019 - -/* REG_MCR */ -#define MCR_APB_96M 7 -#define MCR_APB_166M 12 - -#define WMT_I2C_TIMEOUT (msecs_to_jiffies(1000)) - -struct wmt_i2c_dev { - struct i2c_adapter adapter; - struct completion complete; - struct device *dev; - void __iomem *base; - struct clk *clk; - u16 tcr; - int irq; - u16 cmd_status; -}; - -static int wmt_i2c_wait_bus_not_busy(struct wmt_i2c_dev *i2c_dev) -{ - unsigned long timeout; - - timeout = jiffies + WMT_I2C_TIMEOUT; - while (!(readw(i2c_dev->base + REG_CSR) & CSR_READY_MASK)) { - if (time_after(jiffies, timeout)) { - dev_warn(i2c_dev->dev, "timeout waiting for bus ready\n"); - return -EBUSY; - } - msleep(20); - } - - return 0; -} - -static int wmt_check_status(struct wmt_i2c_dev *i2c_dev) -{ - int ret = 0; - unsigned long wait_result; - - wait_result = wait_for_completion_timeout(&i2c_dev->complete, - msecs_to_jiffies(500)); - if (!wait_result) - return -ETIMEDOUT; - - if (i2c_dev->cmd_status & ISR_NACK_ADDR) - ret = -EIO; - - if (i2c_dev->cmd_status & ISR_SCL_TIMEOUT) - ret = -ETIMEDOUT; - - return ret; -} - -static int wmt_i2c_write(struct wmt_i2c_dev *i2c_dev, struct i2c_msg *pmsg, - int last) -{ - u16 val, tcr_val = i2c_dev->tcr; - int ret; - int xfer_len = 0; - - if (pmsg->len == 0) { - /* - * We still need to run through the while (..) once, so - * start at -1 and break out early from the loop - */ - xfer_len = -1; - writew(0, i2c_dev->base + REG_CDR); - } else { - writew(pmsg->buf[0] & 0xFF, i2c_dev->base + REG_CDR); - } - - if (!(pmsg->flags & I2C_M_NOSTART)) { - val = readw(i2c_dev->base + REG_CR); - val &= ~CR_TX_END; - val |= CR_CPU_RDY; - writew(val, i2c_dev->base + REG_CR); - } - - reinit_completion(&i2c_dev->complete); - - tcr_val |= (TCR_MASTER_WRITE | (pmsg->addr & TCR_SLAVE_ADDR_MASK)); - - writew(tcr_val, i2c_dev->base + REG_TCR); - - if (pmsg->flags & I2C_M_NOSTART) { - val = readw(i2c_dev->base + REG_CR); - val |= CR_CPU_RDY; - writew(val, i2c_dev->base + REG_CR); - } - - while (xfer_len < pmsg->len) { - ret = wmt_check_status(i2c_dev); - if (ret) - return ret; - - xfer_len++; - - val = readw(i2c_dev->base + REG_CSR); - if ((val & CSR_RCV_ACK_MASK) == CSR_RCV_NOT_ACK) { - dev_dbg(i2c_dev->dev, "write RCV NACK error\n"); - return -EIO; - } - - if (pmsg->len == 0) { - val = CR_TX_END | CR_CPU_RDY | CR_ENABLE; - writew(val, i2c_dev->base + REG_CR); - break; - } - - if (xfer_len == pmsg->len) { - if (last != 1) - writew(CR_ENABLE, i2c_dev->base + REG_CR); - } else { - writew(pmsg->buf[xfer_len] & 0xFF, i2c_dev->base + - REG_CDR); - writew(CR_CPU_RDY | CR_ENABLE, i2c_dev->base + REG_CR); - } - } - - return 0; -} - -static int wmt_i2c_read(struct wmt_i2c_dev *i2c_dev, struct i2c_msg *pmsg) -{ - u16 val, tcr_val = i2c_dev->tcr; - int ret; - u32 xfer_len = 0; - - val = readw(i2c_dev->base + REG_CR); - val &= ~(CR_TX_END | CR_TX_NEXT_NO_ACK); - - if (!(pmsg->flags & I2C_M_NOSTART)) - val |= CR_CPU_RDY; - - if (pmsg->len == 1) - val |= CR_TX_NEXT_NO_ACK; - - writew(val, i2c_dev->base + REG_CR); - - reinit_completion(&i2c_dev->complete); - - tcr_val |= TCR_MASTER_READ | (pmsg->addr & TCR_SLAVE_ADDR_MASK); - - writew(tcr_val, i2c_dev->base + REG_TCR); - - if (pmsg->flags & I2C_M_NOSTART) { - val = readw(i2c_dev->base + REG_CR); - val |= CR_CPU_RDY; - writew(val, i2c_dev->base + REG_CR); - } - - while (xfer_len < pmsg->len) { - ret = wmt_check_status(i2c_dev); - if (ret) - return ret; - - pmsg->buf[xfer_len] = readw(i2c_dev->base + REG_CDR) >> 8; - xfer_len++; - - val = readw(i2c_dev->base + REG_CR) | CR_CPU_RDY; - if (xfer_len == pmsg->len - 1) - val |= CR_TX_NEXT_NO_ACK; - writew(val, i2c_dev->base + REG_CR); - } - - return 0; -} - -static int wmt_i2c_xfer(struct i2c_adapter *adap, - struct i2c_msg msgs[], - int num) -{ - struct i2c_msg *pmsg; - int i; - int ret = 0; - struct wmt_i2c_dev *i2c_dev = i2c_get_adapdata(adap); - - for (i = 0; ret >= 0 && i < num; i++) { - pmsg = &msgs[i]; - if (!(pmsg->flags & I2C_M_NOSTART)) { - ret = wmt_i2c_wait_bus_not_busy(i2c_dev); - if (ret < 0) - return ret; - } - - if (pmsg->flags & I2C_M_RD) - ret = wmt_i2c_read(i2c_dev, pmsg); - else - ret = wmt_i2c_write(i2c_dev, pmsg, (i + 1) == num); - } - - return (ret < 0) ? ret : i; -} - -static u32 wmt_i2c_func(struct i2c_adapter *adap) -{ - return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | I2C_FUNC_NOSTART; -} - -static const struct i2c_algorithm wmt_i2c_algo = { - .master_xfer = wmt_i2c_xfer, - .functionality = wmt_i2c_func, -}; - -static irqreturn_t wmt_i2c_isr(int irq, void *data) -{ - struct wmt_i2c_dev *i2c_dev = data; - - /* save the status and write-clear it */ - i2c_dev->cmd_status = readw(i2c_dev->base + REG_ISR); - writew(i2c_dev->cmd_status, i2c_dev->base + REG_ISR); - - complete(&i2c_dev->complete); - - return IRQ_HANDLED; -} - -static int wmt_i2c_reset_hardware(struct wmt_i2c_dev *i2c_dev) -{ - int err; - - err = clk_prepare_enable(i2c_dev->clk); - if (err) { - dev_err(i2c_dev->dev, "failed to enable clock\n"); - return err; - } - - err = clk_set_rate(i2c_dev->clk, 20000000); - if (err) { - dev_err(i2c_dev->dev, "failed to set clock = 20Mhz\n"); - clk_disable_unprepare(i2c_dev->clk); - return err; - } - - writew(0, i2c_dev->base + REG_CR); - writew(MCR_APB_166M, i2c_dev->base + REG_MCR); - writew(ISR_WRITE_ALL, i2c_dev->base + REG_ISR); - writew(IMR_ENABLE_ALL, i2c_dev->base + REG_IMR); - writew(CR_ENABLE, i2c_dev->base + REG_CR); - readw(i2c_dev->base + REG_CSR); /* read clear */ - writew(ISR_WRITE_ALL, i2c_dev->base + REG_ISR); - - if (i2c_dev->tcr == TCR_FAST_MODE) - writew(SCL_TIMEOUT(128) | TR_HS, i2c_dev->base + REG_TR); - else - writew(SCL_TIMEOUT(128) | TR_STD, i2c_dev->base + REG_TR); - - return 0; -} - -static int wmt_i2c_probe(struct platform_device *pdev) -{ - struct device_node *np = pdev->dev.of_node; - struct wmt_i2c_dev *i2c_dev; - struct i2c_adapter *adap; - int err; - u32 clk_rate; - - i2c_dev = devm_kzalloc(&pdev->dev, sizeof(*i2c_dev), GFP_KERNEL); - if (!i2c_dev) - return -ENOMEM; - - i2c_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); - if (IS_ERR(i2c_dev->base)) - return PTR_ERR(i2c_dev->base); - - i2c_dev->irq = irq_of_parse_and_map(np, 0); - if (!i2c_dev->irq) { - dev_err(&pdev->dev, "irq missing or invalid\n"); - return -EINVAL; - } - - i2c_dev->clk = of_clk_get(np, 0); - if (IS_ERR(i2c_dev->clk)) { - dev_err(&pdev->dev, "unable to request clock\n"); - return PTR_ERR(i2c_dev->clk); - } - - err = of_property_read_u32(np, "clock-frequency", &clk_rate); - if (!err && (clk_rate == I2C_MAX_FAST_MODE_FREQ)) - i2c_dev->tcr = TCR_FAST_MODE; - - i2c_dev->dev = &pdev->dev; - - err = devm_request_irq(&pdev->dev, i2c_dev->irq, wmt_i2c_isr, 0, - "i2c", i2c_dev); - if (err) { - dev_err(&pdev->dev, "failed to request irq %i\n", i2c_dev->irq); - return err; - } - - adap = &i2c_dev->adapter; - i2c_set_adapdata(adap, i2c_dev); - strscpy(adap->name, "WMT I2C adapter", sizeof(adap->name)); - adap->owner = THIS_MODULE; - adap->algo = &wmt_i2c_algo; - adap->dev.parent = &pdev->dev; - adap->dev.of_node = pdev->dev.of_node; - - init_completion(&i2c_dev->complete); - - err = wmt_i2c_reset_hardware(i2c_dev); - if (err) { - dev_err(&pdev->dev, "error initializing hardware\n"); - return err; - } - - err = i2c_add_adapter(adap); - if (err) - goto err_disable_clk; - - platform_set_drvdata(pdev, i2c_dev); - - return 0; - -err_disable_clk: - clk_disable_unprepare(i2c_dev->clk); - return err; -} - -static void wmt_i2c_remove(struct platform_device *pdev) -{ - struct wmt_i2c_dev *i2c_dev = platform_get_drvdata(pdev); - - /* Disable interrupts, clock and delete adapter */ - writew(0, i2c_dev->base + REG_IMR); - clk_disable_unprepare(i2c_dev->clk); - i2c_del_adapter(&i2c_dev->adapter); -} - -static const struct of_device_id wmt_i2c_dt_ids[] = { - { .compatible = "wm,wm8505-i2c" }, - { /* Sentinel */ }, -}; - -static struct platform_driver wmt_i2c_driver = { - .probe = wmt_i2c_probe, - .remove_new = wmt_i2c_remove, - .driver = { - .name = "wmt-i2c", - .of_match_table = wmt_i2c_dt_ids, - }, -}; - -module_platform_driver(wmt_i2c_driver); - -MODULE_DESCRIPTION("Wondermedia I2C master-mode bus adapter"); -MODULE_AUTHOR("Tony Prisk <linux@prisktech.co.nz>"); -MODULE_LICENSE("GPL"); -MODULE_DEVICE_TABLE(of, wmt_i2c_dt_ids); diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c index d6037a328669..14ae0cfc325e 100644 --- a/drivers/i2c/i2c-core-acpi.c +++ b/drivers/i2c/i2c-core-acpi.c @@ -445,6 +445,11 @@ static struct i2c_client *i2c_acpi_find_client_by_adev(struct acpi_device *adev) return i2c_find_device_by_fwnode(acpi_fwnode_handle(adev)); } +static struct i2c_adapter *i2c_acpi_find_adapter_by_adev(struct acpi_device *adev) +{ + return i2c_find_adapter_by_fwnode(acpi_fwnode_handle(adev)); +} + static int i2c_acpi_notify(struct notifier_block *nb, unsigned long value, void *arg) { @@ -471,11 +476,17 @@ static int i2c_acpi_notify(struct notifier_block *nb, unsigned long value, break; client = i2c_acpi_find_client_by_adev(adev); - if (!client) - break; + if (client) { + i2c_unregister_device(client); + put_device(&client->dev); + } + + adapter = i2c_acpi_find_adapter_by_adev(adev); + if (adapter) { + acpi_unbind_one(&adapter->dev); + put_device(&adapter->dev); + } - i2c_unregister_device(client); - put_device(&client->dev); break; } diff --git a/drivers/i2c/i2c-mux.c b/drivers/i2c/i2c-mux.c index 57ff09f18c37..fda72e8be885 100644 --- a/drivers/i2c/i2c-mux.c +++ b/drivers/i2c/i2c-mux.c @@ -127,19 +127,6 @@ static u32 i2c_mux_functionality(struct i2c_adapter *adap) return parent->algo->functionality(parent); } -/* Return all parent classes, merged */ -static unsigned int i2c_mux_parent_classes(struct i2c_adapter *parent) -{ - unsigned int class = 0; - - do { - class |= parent->class; - parent = i2c_parent_is_i2c_adapter(parent); - } while (parent); - - return class; -} - static void i2c_mux_lock_bus(struct i2c_adapter *adapter, unsigned int flags) { struct i2c_mux_priv *priv = adapter->algo_data; @@ -281,8 +268,7 @@ static const struct i2c_lock_operations i2c_parent_lock_ops = { }; int i2c_mux_add_adapter(struct i2c_mux_core *muxc, - u32 force_nr, u32 chan_id, - unsigned int class) + u32 force_nr, u32 chan_id) { struct i2c_adapter *parent = muxc->parent; struct i2c_mux_priv *priv; @@ -340,14 +326,6 @@ int i2c_mux_add_adapter(struct i2c_mux_core *muxc, else priv->adap.lock_ops = &i2c_parent_lock_ops; - /* Sanity check on class */ - if (i2c_mux_parent_classes(parent) & class & ~I2C_CLASS_DEPRECATED) - dev_err(&parent->dev, - "Segment %d behind mux can't share classes with ancestors\n", - chan_id); - else - priv->adap.class = class; - /* * Try to populate the mux adapter's of_node, expands to * nothing if !CONFIG_OF. diff --git a/drivers/i2c/muxes/i2c-arb-gpio-challenge.c b/drivers/i2c/muxes/i2c-arb-gpio-challenge.c index 24168e9f7df4..7aa6e795d833 100644 --- a/drivers/i2c/muxes/i2c-arb-gpio-challenge.c +++ b/drivers/i2c/muxes/i2c-arb-gpio-challenge.c @@ -167,7 +167,7 @@ static int i2c_arbitrator_probe(struct platform_device *pdev) } /* Actually add the mux adapter */ - ret = i2c_mux_add_adapter(muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(muxc, 0, 0); if (ret) i2c_put_adapter(muxc->parent); diff --git a/drivers/i2c/muxes/i2c-mux-gpio.c b/drivers/i2c/muxes/i2c-mux-gpio.c index 6b979a0a6ab8..d6bbb8b68333 100644 --- a/drivers/i2c/muxes/i2c-mux-gpio.c +++ b/drivers/i2c/muxes/i2c-mux-gpio.c @@ -206,9 +206,8 @@ static int i2c_mux_gpio_probe(struct platform_device *pdev) for (i = 0; i < mux->data.n_values; i++) { u32 nr = mux->data.base_nr ? (mux->data.base_nr + i) : 0; - unsigned int class = mux->data.classes ? mux->data.classes[i] : 0; - ret = i2c_mux_add_adapter(muxc, nr, mux->data.values[i], class); + ret = i2c_mux_add_adapter(muxc, nr, mux->data.values[i]); if (ret) goto add_adapter_failed; } diff --git a/drivers/i2c/muxes/i2c-mux-gpmux.c b/drivers/i2c/muxes/i2c-mux-gpmux.c index 8305661e1253..10d63307b14d 100644 --- a/drivers/i2c/muxes/i2c-mux-gpmux.c +++ b/drivers/i2c/muxes/i2c-mux-gpmux.c @@ -124,7 +124,7 @@ static int i2c_mux_probe(struct platform_device *pdev) goto err_children; } - ret = i2c_mux_add_adapter(muxc, 0, chan, 0); + ret = i2c_mux_add_adapter(muxc, 0, chan); if (ret) goto err_children; } diff --git a/drivers/i2c/muxes/i2c-mux-ltc4306.c b/drivers/i2c/muxes/i2c-mux-ltc4306.c index 23766d853e76..19a7c370946d 100644 --- a/drivers/i2c/muxes/i2c-mux-ltc4306.c +++ b/drivers/i2c/muxes/i2c-mux-ltc4306.c @@ -279,7 +279,7 @@ static int ltc4306_probe(struct i2c_client *client) /* Now create an adapter for each channel */ for (num = 0; num < chip->nchans; num++) { - ret = i2c_mux_add_adapter(muxc, 0, num, 0); + ret = i2c_mux_add_adapter(muxc, 0, num); if (ret) { i2c_mux_del_adapters(muxc); return ret; diff --git a/drivers/i2c/muxes/i2c-mux-mlxcpld.c b/drivers/i2c/muxes/i2c-mux-mlxcpld.c index 4c6ed1d58c79..3f06aa3331a7 100644 --- a/drivers/i2c/muxes/i2c-mux-mlxcpld.c +++ b/drivers/i2c/muxes/i2c-mux-mlxcpld.c @@ -154,7 +154,7 @@ static int mlxcpld_mux_probe(struct platform_device *pdev) /* Create an adapter for each channel. */ for (num = 0; num < pdata->num_adaps; num++) { - err = i2c_mux_add_adapter(muxc, 0, pdata->chan_ids[num], 0); + err = i2c_mux_add_adapter(muxc, 0, pdata->chan_ids[num]); if (err) goto virt_reg_failed; } diff --git a/drivers/i2c/muxes/i2c-mux-pca9541.c b/drivers/i2c/muxes/i2c-mux-pca9541.c index ce0fb69249a8..e28694d991fb 100644 --- a/drivers/i2c/muxes/i2c-mux-pca9541.c +++ b/drivers/i2c/muxes/i2c-mux-pca9541.c @@ -314,7 +314,7 @@ static int pca9541_probe(struct i2c_client *client) i2c_set_clientdata(client, muxc); - ret = i2c_mux_add_adapter(muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(muxc, 0, 0); if (ret) return ret; diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c index c3f4ff08ac38..6f84018258c4 100644 --- a/drivers/i2c/muxes/i2c-mux-pca954x.c +++ b/drivers/i2c/muxes/i2c-mux-pca954x.c @@ -644,7 +644,7 @@ static int pca954x_probe(struct i2c_client *client) /* Now create an adapter for each channel */ for (num = 0; num < data->chip->nchans; num++) { - ret = i2c_mux_add_adapter(muxc, 0, num, 0); + ret = i2c_mux_add_adapter(muxc, 0, num); if (ret) goto fail_cleanup; } diff --git a/drivers/i2c/muxes/i2c-mux-pinctrl.c b/drivers/i2c/muxes/i2c-mux-pinctrl.c index 6ebca7bfd8a2..02aaf0781e9c 100644 --- a/drivers/i2c/muxes/i2c-mux-pinctrl.c +++ b/drivers/i2c/muxes/i2c-mux-pinctrl.c @@ -151,7 +151,7 @@ static int i2c_mux_pinctrl_probe(struct platform_device *pdev) /* Do not add any adapter for the idle state (if it's there at all). */ for (i = 0; i < num_names - !!muxc->deselect; i++) { - ret = i2c_mux_add_adapter(muxc, 0, i, 0); + ret = i2c_mux_add_adapter(muxc, 0, i); if (ret) goto err_del_adapter; } diff --git a/drivers/i2c/muxes/i2c-mux-reg.c b/drivers/i2c/muxes/i2c-mux-reg.c index 8489971babd3..ef765fcd33f5 100644 --- a/drivers/i2c/muxes/i2c-mux-reg.c +++ b/drivers/i2c/muxes/i2c-mux-reg.c @@ -213,7 +213,7 @@ static int i2c_mux_reg_probe(struct platform_device *pdev) for (i = 0; i < mux->data.n_values; i++) { nr = mux->data.base_nr ? (mux->data.base_nr + i) : 0; - ret = i2c_mux_add_adapter(muxc, nr, mux->data.values[i], 0); + ret = i2c_mux_add_adapter(muxc, nr, mux->data.values[i]); if (ret) goto err_del_mux_adapters; } diff --git a/drivers/iio/gyro/mpu3050-i2c.c b/drivers/iio/gyro/mpu3050-i2c.c index 52b6feed2637..29ecfa6fd633 100644 --- a/drivers/iio/gyro/mpu3050-i2c.c +++ b/drivers/iio/gyro/mpu3050-i2c.c @@ -72,7 +72,7 @@ static int mpu3050_i2c_probe(struct i2c_client *client) else { mpu3050->i2cmux->priv = mpu3050; /* Ignore failure, not critical */ - i2c_mux_add_adapter(mpu3050->i2cmux, 0, 0, 0); + i2c_mux_add_adapter(mpu3050->i2cmux, 0, 0); } return 0; diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c index 410ea39fd495..0e03137fb3d4 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c @@ -142,7 +142,7 @@ static int inv_mpu_probe(struct i2c_client *client) if (!st->muxc) return -ENOMEM; st->muxc->priv = dev_get_drvdata(&client->dev); - result = i2c_mux_add_adapter(st->muxc, 0, 0, 0); + result = i2c_mux_add_adapter(st->muxc, 0, 0); if (result) return result; result = inv_mpu_acpi_create_mux_client(client); diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 1e2cd7c8716e..64ace0b968f0 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -715,8 +715,10 @@ cma_validate_port(struct ib_device *device, u32 port, rcu_read_lock(); ndev = rcu_dereference(sgid_attr->ndev); if (!net_eq(dev_net(ndev), dev_addr->net) || - ndev->ifindex != bound_if_index) + ndev->ifindex != bound_if_index) { + rdma_put_gid_attr(sgid_attr); sgid_attr = ERR_PTR(-ENODEV); + } rcu_read_unlock(); goto out; } diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 07cb6c5ffda0..55aa7aa32d4a 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -2174,8 +2174,7 @@ int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev, spin_unlock_irqrestore(&pdata->netdev_lock, flags); add_ndev_hash(pdata); - if (old_ndev) - __dev_put(old_ndev); + __dev_put(old_ndev); return 0; } @@ -2235,8 +2234,7 @@ struct net_device *ib_device_get_netdev(struct ib_device *ib_dev, spin_lock(&pdata->netdev_lock); res = rcu_dereference_protected( pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); - if (res) - dev_hold(res); + dev_hold(res); spin_unlock(&pdata->netdev_lock); } @@ -2311,9 +2309,7 @@ void ib_enum_roce_netdev(struct ib_device *ib_dev, if (filter(ib_dev, port, idev, filter_cookie)) cb(ib_dev, port, idev, cookie); - - if (idev) - dev_put(idev); + dev_put(idev); } } diff --git a/drivers/infiniband/core/lag.c b/drivers/infiniband/core/lag.c index eca6e37c72ba..8fd80adfe833 100644 --- a/drivers/infiniband/core/lag.c +++ b/drivers/infiniband/core/lag.c @@ -93,8 +93,7 @@ static struct net_device *rdma_get_xmit_slave_udp(struct ib_device *device, slave = netdev_get_xmit_slave(master, skb, !!(device->lag_flags & RDMA_LAG_FLAGS_HASH_ALL_SLAVES)); - if (slave) - dev_hold(slave); + dev_hold(slave); rcu_read_unlock(); kfree_skb(skb); return slave; diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 4900a0848124..bc79ee630d8d 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -137,6 +137,8 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME]= { .type = NLA_NUL_STRING, .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, [RDMA_NLDEV_ATTR_RES_TYPE] = { .type = NLA_U8 }, + [RDMA_NLDEV_ATTR_RES_SUBTYPE] = { .type = NLA_NUL_STRING, + .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, [RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]= { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_RES_USECNT] = { .type = NLA_U64 }, [RDMA_NLDEV_ATTR_RES_SRQ] = { .type = NLA_NESTED }, @@ -164,6 +166,7 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC] = { .type = NLA_U8 }, [RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE] = { .type = NLA_U8 }, + [RDMA_NLDEV_ATTR_DRIVER_DETAILS] = { .type = NLA_U8 }, }; static int put_driver_name_print_type(struct sk_buff *msg, const char *name, @@ -399,7 +402,8 @@ err: return -EMSGSIZE; } -static int fill_res_info(struct sk_buff *msg, struct ib_device *device) +static int fill_res_info(struct sk_buff *msg, struct ib_device *device, + bool show_details) { static const char * const names[RDMA_RESTRACK_MAX] = { [RDMA_RESTRACK_PD] = "pd", @@ -424,7 +428,7 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device) for (i = 0; i < RDMA_RESTRACK_MAX; i++) { if (!names[i]) continue; - curr = rdma_restrack_count(device, i); + curr = rdma_restrack_count(device, i, show_details); ret = fill_res_info_entry(msg, names[i], curr); if (ret) goto err; @@ -1305,6 +1309,7 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, struct netlink_ext_ack *extack) { struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; + bool show_details = false; struct ib_device *device; struct sk_buff *msg; u32 index; @@ -1320,6 +1325,9 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, if (!device) return -EINVAL; + if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]) + show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]); + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); if (!msg) { ret = -ENOMEM; @@ -1334,7 +1342,7 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, goto err_free; } - ret = fill_res_info(msg, device); + ret = fill_res_info(msg, device, show_details); if (ret) goto err_free; @@ -1364,7 +1372,7 @@ static int _nldev_res_get_dumpit(struct ib_device *device, RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET), 0, NLM_F_MULTI); - if (!nlh || fill_res_info(skb, device)) { + if (!nlh || fill_res_info(skb, device, false)) { nlmsg_cancel(skb, nlh); goto out; } @@ -1534,6 +1542,7 @@ static int res_get_common_dumpit(struct sk_buff *skb, struct rdma_restrack_entry *res; struct rdma_restrack_root *rt; int err, ret = 0, idx = 0; + bool show_details = false; struct nlattr *table_attr; struct nlattr *entry_attr; struct ib_device *device; @@ -1562,6 +1571,9 @@ static int res_get_common_dumpit(struct sk_buff *skb, if (!device) return -EINVAL; + if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]) + show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]); + /* * If no PORT_INDEX is supplied, we will return all QPs from that device */ @@ -1599,6 +1611,9 @@ static int res_get_common_dumpit(struct sk_buff *skb, * objects. */ xa_for_each(&rt->xa, id, res) { + if (xa_get_mark(&rt->xa, res->id, RESTRACK_DD) && !show_details) + goto next; + if (idx < start || !rdma_restrack_get(res)) goto next; diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index 01a499a8b88d..3313410014cd 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -37,22 +37,6 @@ int rdma_restrack_init(struct ib_device *dev) return 0; } -static const char *type2str(enum rdma_restrack_type type) -{ - static const char * const names[RDMA_RESTRACK_MAX] = { - [RDMA_RESTRACK_PD] = "PD", - [RDMA_RESTRACK_CQ] = "CQ", - [RDMA_RESTRACK_QP] = "QP", - [RDMA_RESTRACK_CM_ID] = "CM_ID", - [RDMA_RESTRACK_MR] = "MR", - [RDMA_RESTRACK_CTX] = "CTX", - [RDMA_RESTRACK_COUNTER] = "COUNTER", - [RDMA_RESTRACK_SRQ] = "SRQ", - }; - - return names[type]; -}; - /** * rdma_restrack_clean() - clean resource tracking * @dev: IB device @@ -60,47 +44,14 @@ static const char *type2str(enum rdma_restrack_type type) void rdma_restrack_clean(struct ib_device *dev) { struct rdma_restrack_root *rt = dev->res; - struct rdma_restrack_entry *e; - char buf[TASK_COMM_LEN]; - bool found = false; - const char *owner; int i; for (i = 0 ; i < RDMA_RESTRACK_MAX; i++) { struct xarray *xa = &dev->res[i].xa; - if (!xa_empty(xa)) { - unsigned long index; - - if (!found) { - pr_err("restrack: %s", CUT_HERE); - dev_err(&dev->dev, "BUG: RESTRACK detected leak of resources\n"); - } - xa_for_each(xa, index, e) { - if (rdma_is_kernel_res(e)) { - owner = e->kern_name; - } else { - /* - * There is no need to call get_task_struct here, - * because we can be here only if there are more - * get_task_struct() call than put_task_struct(). - */ - get_task_comm(buf, e->task); - owner = buf; - } - - pr_err("restrack: %s %s object allocated by %s is not freed\n", - rdma_is_kernel_res(e) ? "Kernel" : - "User", - type2str(e->type), owner); - } - found = true; - } + WARN_ON(!xa_empty(xa)); xa_destroy(xa); } - if (found) - pr_err("restrack: %s", CUT_HERE); - kfree(rt); } @@ -108,8 +59,10 @@ void rdma_restrack_clean(struct ib_device *dev) * rdma_restrack_count() - the current usage of specific object * @dev: IB device * @type: actual type of object to operate + * @show_details: count driver specific objects */ -int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type) +int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, + bool show_details) { struct rdma_restrack_root *rt = &dev->res[type]; struct rdma_restrack_entry *e; @@ -117,8 +70,11 @@ int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type) u32 cnt = 0; xa_lock(&rt->xa); - xas_for_each(&xas, e, U32_MAX) + xas_for_each(&xas, e, U32_MAX) { + if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details) + continue; cnt++; + } xa_unlock(&rt->xa); return cnt; } @@ -247,6 +203,9 @@ void rdma_restrack_add(struct rdma_restrack_entry *res) ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL); if (ret) res->id = 0; + + if (qp->qp_type >= IB_QPT_DRIVER) + xa_set_mark(&rt->xa, res->id, RESTRACK_DD); } else if (res->type == RDMA_RESTRACK_COUNTER) { /* Special case to ensure that cntn points to right counter */ struct rdma_counter *counter; diff --git a/drivers/infiniband/core/roce_gid_mgmt.c b/drivers/infiniband/core/roce_gid_mgmt.c index e958c43dd28f..d5131b3ba8ab 100644 --- a/drivers/infiniband/core/roce_gid_mgmt.c +++ b/drivers/infiniband/core/roce_gid_mgmt.c @@ -601,8 +601,7 @@ static void del_netdev_default_ips_join(struct ib_device *ib_dev, u32 port, rcu_read_lock(); master_ndev = netdev_master_upper_dev_get_rcu(rdma_ndev); - if (master_ndev) - dev_hold(master_ndev); + dev_hold(master_ndev); rcu_read_unlock(); if (master_ndev) { diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c index f5feca7fa9b9..2ed749f50a29 100644 --- a/drivers/infiniband/core/user_mad.c +++ b/drivers/infiniband/core/user_mad.c @@ -63,6 +63,8 @@ MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand userspace MAD packet access"); MODULE_LICENSE("Dual BSD/GPL"); +#define MAX_UMAD_RECV_LIST_SIZE 200000 + enum { IB_UMAD_MAX_PORTS = RDMA_MAX_PORTS, IB_UMAD_MAX_AGENTS = 32, @@ -113,6 +115,7 @@ struct ib_umad_file { struct mutex mutex; struct ib_umad_port *port; struct list_head recv_list; + atomic_t recv_list_size; struct list_head send_list; struct list_head port_list; spinlock_t send_lock; @@ -180,24 +183,28 @@ static struct ib_mad_agent *__get_agent(struct ib_umad_file *file, int id) return file->agents_dead ? NULL : file->agent[id]; } -static int queue_packet(struct ib_umad_file *file, - struct ib_mad_agent *agent, - struct ib_umad_packet *packet) +static int queue_packet(struct ib_umad_file *file, struct ib_mad_agent *agent, + struct ib_umad_packet *packet, bool is_recv_mad) { int ret = 1; mutex_lock(&file->mutex); + if (is_recv_mad && + atomic_read(&file->recv_list_size) > MAX_UMAD_RECV_LIST_SIZE) + goto unlock; + for (packet->mad.hdr.id = 0; packet->mad.hdr.id < IB_UMAD_MAX_AGENTS; packet->mad.hdr.id++) if (agent == __get_agent(file, packet->mad.hdr.id)) { list_add_tail(&packet->list, &file->recv_list); + atomic_inc(&file->recv_list_size); wake_up_interruptible(&file->recv_wait); ret = 0; break; } - +unlock: mutex_unlock(&file->mutex); return ret; @@ -224,7 +231,7 @@ static void send_handler(struct ib_mad_agent *agent, if (send_wc->status == IB_WC_RESP_TIMEOUT_ERR) { packet->length = IB_MGMT_MAD_HDR; packet->mad.hdr.status = ETIMEDOUT; - if (!queue_packet(file, agent, packet)) + if (!queue_packet(file, agent, packet, false)) return; } kfree(packet); @@ -284,7 +291,7 @@ static void recv_handler(struct ib_mad_agent *agent, rdma_destroy_ah_attr(&ah_attr); } - if (queue_packet(file, agent, packet)) + if (queue_packet(file, agent, packet, true)) goto err2; return; @@ -409,6 +416,7 @@ static ssize_t ib_umad_read(struct file *filp, char __user *buf, packet = list_entry(file->recv_list.next, struct ib_umad_packet, list); list_del(&packet->list); + atomic_dec(&file->recv_list_size); mutex_unlock(&file->mutex); @@ -421,6 +429,7 @@ static ssize_t ib_umad_read(struct file *filp, char __user *buf, /* Requeue packet */ mutex_lock(&file->mutex); list_add(&packet->list, &file->recv_list); + atomic_inc(&file->recv_list_size); mutex_unlock(&file->mutex); } else { if (packet->recv_wc) diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index 439d0c7c5d0c..04258676d072 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -1013,7 +1013,8 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) hwq_attr.stride = sizeof(struct sq_sge); hwq_attr.depth = bnxt_qplib_get_depth(sq); hwq_attr.aux_stride = psn_sz; - hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode); + hwq_attr.aux_depth = psn_sz ? bnxt_qplib_set_sq_size(sq, qp->wqe_mode) + : 0; /* Update msn tbl size */ if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { hwq_attr.aux_depth = roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h index 7377c8a9f4d5..4296662e59c3 100644 --- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h @@ -110,7 +110,10 @@ struct efa_admin_create_qp_cmd { * virtual (IOVA returned by MR registration) * 1 : rq_virt - If set, RQ ring base address is * virtual (IOVA returned by MR registration) - * 7:2 : reserved - MBZ + * 2 : unsolicited_write_recv - If set, work requests + * will not be consumed for incoming RDMA write with + * immediate + * 7:3 : reserved - MBZ */ u8 flags; @@ -663,7 +666,9 @@ struct efa_admin_feature_device_attr_desc { * polling is supported * 3 : rdma_write - If set, RDMA Write is supported * on TX queues - * 31:4 : reserved - MBZ + * 4 : unsolicited_write_recv - If set, unsolicited + * write with imm. receive is supported + * 31:5 : reserved - MBZ */ u32 device_caps; @@ -1009,6 +1014,7 @@ struct efa_admin_host_info { /* create_qp_cmd */ #define EFA_ADMIN_CREATE_QP_CMD_SQ_VIRT_MASK BIT(0) #define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_MASK BIT(1) +#define EFA_ADMIN_CREATE_QP_CMD_UNSOLICITED_WRITE_RECV_MASK BIT(2) /* modify_qp_cmd */ #define EFA_ADMIN_MODIFY_QP_CMD_QP_STATE_MASK BIT(0) @@ -1044,6 +1050,7 @@ struct efa_admin_host_info { #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RNR_RETRY_MASK BIT(1) #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_DATA_POLLING_128_MASK BIT(2) #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK BIT(3) +#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4) /* create_eq_cmd */ #define EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS_MASK GENMASK(4, 0) diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index d3398c7b0bd0..5b9c2b16df0e 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -32,6 +32,9 @@ int efa_com_create_qp(struct efa_com_dev *edev, params->rq_depth; create_qp_cmd.uar = params->uarn; + if (params->unsolicited_write_recv) + EFA_SET(&create_qp_cmd.flags, EFA_ADMIN_CREATE_QP_CMD_UNSOLICITED_WRITE_RECV, 1); + err = efa_com_cmd_exec(aq, (struct efa_admin_aq_entry *)&create_qp_cmd, sizeof(create_qp_cmd), diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h index 720a99ba0f7d..9714105fcf7e 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.h +++ b/drivers/infiniband/hw/efa/efa_com_cmd.h @@ -27,6 +27,7 @@ struct efa_com_create_qp_params { u16 pd; u16 uarn; u8 qp_type; + u8 unsolicited_write_recv : 1; }; struct efa_com_create_qp_result { diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 5fa3603c80d8..d1a48f988f6c 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -671,11 +671,22 @@ static void efa_remove(struct pci_dev *pdev) efa_remove_device(pdev); } +static void efa_shutdown(struct pci_dev *pdev) +{ + struct efa_dev *dev = pci_get_drvdata(pdev); + + efa_destroy_eqs(dev); + efa_com_dev_reset(&dev->edev, EFA_REGS_RESET_SHUTDOWN); + efa_free_irq(dev, &dev->admin_irq); + efa_disable_msix(dev); +} + static struct pci_driver efa_pci_driver = { .name = DRV_MODULE_NAME, .id_table = efa_pci_tbl, .probe = efa_probe, .remove = efa_remove, + .shutdown = efa_shutdown, }; module_pci_driver(efa_pci_driver); diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 2f412db2edcd..8f7a13b79cdc 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -263,6 +263,9 @@ int efa_query_device(struct ib_device *ibdev, if (EFA_DEV_CAP(dev, RDMA_WRITE)) resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_WRITE; + if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV)) + resp.device_caps |= EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV; + if (dev->neqs) resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS; @@ -639,6 +642,7 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, struct efa_ibv_create_qp cmd = {}; struct efa_qp *qp = to_eqp(ibqp); struct efa_ucontext *ucontext; + u16 supported_efa_flags = 0; int err; ucontext = rdma_udata_to_drv_context(udata, struct efa_ucontext, @@ -676,13 +680,23 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, goto err_out; } - if (cmd.comp_mask) { + if (cmd.comp_mask || !is_reserved_cleared(cmd.reserved_90)) { ibdev_dbg(&dev->ibdev, "Incompatible ABI params, unknown fields in udata\n"); err = -EINVAL; goto err_out; } + if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV)) + supported_efa_flags |= EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV; + + if (cmd.flags & ~supported_efa_flags) { + ibdev_dbg(&dev->ibdev, "Unsupported EFA QP create flags[%#x], supported[%#x]\n", + cmd.flags, supported_efa_flags); + err = -EOPNOTSUPP; + goto err_out; + } + create_qp_params.uarn = ucontext->uarn; create_qp_params.pd = to_epd(ibqp->pd)->pdn; @@ -722,6 +736,9 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, create_qp_params.rq_base_addr = qp->rq_dma_addr; } + if (cmd.flags & EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV) + create_qp_params.unsolicited_write_recv = true; + err = efa_com_create_qp(&dev->edev, &create_qp_params, &create_qp_resp); if (err) diff --git a/drivers/infiniband/hw/erdma/erdma.h b/drivers/infiniband/hw/erdma/erdma.h index 5df401a30cb9..c8bd698e21b0 100644 --- a/drivers/infiniband/hw/erdma/erdma.h +++ b/drivers/infiniband/hw/erdma/erdma.h @@ -33,7 +33,8 @@ struct erdma_eq { atomic64_t notify_num; void __iomem *db; - u64 *db_record; + u64 *dbrec; + dma_addr_t dbrec_dma; }; struct erdma_cmdq_sq { @@ -48,7 +49,8 @@ struct erdma_cmdq_sq { u16 wqebb_cnt; - u64 *db_record; + u64 *dbrec; + dma_addr_t dbrec_dma; }; struct erdma_cmdq_cq { @@ -61,7 +63,8 @@ struct erdma_cmdq_cq { u32 ci; u32 cmdsn; - u64 *db_record; + u64 *dbrec; + dma_addr_t dbrec_dma; atomic64_t armed_num; }; @@ -177,9 +180,6 @@ enum { ERDMA_RES_CNT = 2, }; -#define ERDMA_EXTRA_BUFFER_SIZE ERDMA_DB_SIZE -#define WARPPED_BUFSIZE(size) ((size) + ERDMA_EXTRA_BUFFER_SIZE) - struct erdma_dev { struct ib_device ibdev; struct net_device *netdev; @@ -213,6 +213,7 @@ struct erdma_dev { atomic_t num_ctx; struct list_head cep_list; + struct dma_pool *db_pool; struct dma_pool *resp_pool; }; diff --git a/drivers/infiniband/hw/erdma/erdma_cmdq.c b/drivers/infiniband/hw/erdma/erdma_cmdq.c index a151a7bdd504..43ff40b5a09d 100644 --- a/drivers/infiniband/hw/erdma/erdma_cmdq.c +++ b/drivers/infiniband/hw/erdma/erdma_cmdq.c @@ -14,7 +14,7 @@ static void arm_cmdq_cq(struct erdma_cmdq *cmdq) FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cmdq->cq.cmdsn) | FIELD_PREP(ERDMA_CQDB_IDX_MASK, cmdq->cq.cmdsn); - *cmdq->cq.db_record = db_data; + *cmdq->cq.dbrec = db_data; writeq(db_data, dev->func_bar + ERDMA_CMDQ_CQDB_REG); atomic64_inc(&cmdq->cq.armed_num); @@ -25,7 +25,7 @@ static void kick_cmdq_db(struct erdma_cmdq *cmdq) struct erdma_dev *dev = container_of(cmdq, struct erdma_dev, cmdq); u64 db_data = FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq->sq.pi); - *cmdq->sq.db_record = db_data; + *cmdq->sq.dbrec = db_data; writeq(db_data, dev->func_bar + ERDMA_CMDQ_SQDB_REG); } @@ -89,20 +89,18 @@ static int erdma_cmdq_sq_init(struct erdma_dev *dev) { struct erdma_cmdq *cmdq = &dev->cmdq; struct erdma_cmdq_sq *sq = &cmdq->sq; - u32 buf_size; sq->wqebb_cnt = SQEBB_COUNT(ERDMA_CMDQ_SQE_SIZE); sq->depth = cmdq->max_outstandings * sq->wqebb_cnt; - buf_size = sq->depth << SQEBB_SHIFT; - - sq->qbuf = - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), - &sq->qbuf_dma_addr, GFP_KERNEL); + sq->qbuf = dma_alloc_coherent(&dev->pdev->dev, sq->depth << SQEBB_SHIFT, + &sq->qbuf_dma_addr, GFP_KERNEL); if (!sq->qbuf) return -ENOMEM; - sq->db_record = (u64 *)(sq->qbuf + buf_size); + sq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &sq->dbrec_dma); + if (!sq->dbrec) + goto err_out; spin_lock_init(&sq->lock); @@ -111,30 +109,33 @@ static int erdma_cmdq_sq_init(struct erdma_dev *dev) erdma_reg_write32(dev, ERDMA_REGS_CMDQ_SQ_ADDR_L_REG, lower_32_bits(sq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_CMDQ_DEPTH_REG, sq->depth); - erdma_reg_write64(dev, ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG, - sq->qbuf_dma_addr + buf_size); + erdma_reg_write64(dev, ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG, sq->dbrec_dma); return 0; + +err_out: + dma_free_coherent(&dev->pdev->dev, sq->depth << SQEBB_SHIFT, + sq->qbuf, sq->qbuf_dma_addr); + + return -ENOMEM; } static int erdma_cmdq_cq_init(struct erdma_dev *dev) { struct erdma_cmdq *cmdq = &dev->cmdq; struct erdma_cmdq_cq *cq = &cmdq->cq; - u32 buf_size; cq->depth = cmdq->sq.depth; - buf_size = cq->depth << CQE_SHIFT; - - cq->qbuf = - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), - &cq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); + cq->qbuf = dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, + &cq->qbuf_dma_addr, GFP_KERNEL); if (!cq->qbuf) return -ENOMEM; spin_lock_init(&cq->lock); - cq->db_record = (u64 *)(cq->qbuf + buf_size); + cq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &cq->dbrec_dma); + if (!cq->dbrec) + goto err_out; atomic64_set(&cq->armed_num, 0); @@ -142,24 +143,25 @@ static int erdma_cmdq_cq_init(struct erdma_dev *dev) upper_32_bits(cq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_L_REG, lower_32_bits(cq->qbuf_dma_addr)); - erdma_reg_write64(dev, ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG, - cq->qbuf_dma_addr + buf_size); + erdma_reg_write64(dev, ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG, cq->dbrec_dma); return 0; + +err_out: + dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, cq->qbuf, + cq->qbuf_dma_addr); + + return -ENOMEM; } static int erdma_cmdq_eq_init(struct erdma_dev *dev) { struct erdma_cmdq *cmdq = &dev->cmdq; struct erdma_eq *eq = &cmdq->eq; - u32 buf_size; eq->depth = cmdq->max_outstandings; - buf_size = eq->depth << EQE_SHIFT; - - eq->qbuf = - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), - &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); + eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, + &eq->qbuf_dma_addr, GFP_KERNEL); if (!eq->qbuf) return -ENOMEM; @@ -167,17 +169,24 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev) atomic64_set(&eq->event_num, 0); eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG; - eq->db_record = (u64 *)(eq->qbuf + buf_size); + eq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &eq->dbrec_dma); + if (!eq->dbrec) + goto err_out; erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG, upper_32_bits(eq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_L_REG, lower_32_bits(eq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_DEPTH_REG, eq->depth); - erdma_reg_write64(dev, ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG, - eq->qbuf_dma_addr + buf_size); + erdma_reg_write64(dev, ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG, eq->dbrec_dma); return 0; + +err_out: + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, + eq->qbuf_dma_addr); + + return -ENOMEM; } int erdma_cmdq_init(struct erdma_dev *dev) @@ -211,17 +220,17 @@ int erdma_cmdq_init(struct erdma_dev *dev) return 0; err_destroy_cq: - dma_free_coherent(&dev->pdev->dev, - (cmdq->cq.depth << CQE_SHIFT) + - ERDMA_EXTRA_BUFFER_SIZE, + dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT, cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr); + dma_pool_free(dev->db_pool, cmdq->cq.dbrec, cmdq->cq.dbrec_dma); + err_destroy_sq: - dma_free_coherent(&dev->pdev->dev, - (cmdq->sq.depth << SQEBB_SHIFT) + - ERDMA_EXTRA_BUFFER_SIZE, + dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT, cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr); + dma_pool_free(dev->db_pool, cmdq->sq.dbrec, cmdq->sq.dbrec_dma); + return err; } @@ -238,18 +247,20 @@ void erdma_cmdq_destroy(struct erdma_dev *dev) clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state); - dma_free_coherent(&dev->pdev->dev, - (cmdq->eq.depth << EQE_SHIFT) + - ERDMA_EXTRA_BUFFER_SIZE, + dma_free_coherent(&dev->pdev->dev, cmdq->eq.depth << EQE_SHIFT, cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr); - dma_free_coherent(&dev->pdev->dev, - (cmdq->sq.depth << SQEBB_SHIFT) + - ERDMA_EXTRA_BUFFER_SIZE, + + dma_pool_free(dev->db_pool, cmdq->eq.dbrec, cmdq->eq.dbrec_dma); + + dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT, cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr); - dma_free_coherent(&dev->pdev->dev, - (cmdq->cq.depth << CQE_SHIFT) + - ERDMA_EXTRA_BUFFER_SIZE, + + dma_pool_free(dev->db_pool, cmdq->sq.dbrec, cmdq->sq.dbrec_dma); + + dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT, cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr); + + dma_pool_free(dev->db_pool, cmdq->cq.dbrec, cmdq->cq.dbrec_dma); } static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq) diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c index c1cb5568eab2..70f89f0162aa 100644 --- a/drivers/infiniband/hw/erdma/erdma_cq.c +++ b/drivers/infiniband/hw/erdma/erdma_cq.c @@ -26,7 +26,7 @@ static void notify_cq(struct erdma_cq *cq, u8 solcitied) FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cq->kern_cq.cmdsn) | FIELD_PREP(ERDMA_CQDB_CI_MASK, cq->kern_cq.ci); - *cq->kern_cq.db_record = db_data; + *cq->kern_cq.dbrec = db_data; writeq(db_data, cq->kern_cq.db); } diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c index ea47cb21fdb8..84ccdd8144c9 100644 --- a/drivers/infiniband/hw/erdma/erdma_eq.c +++ b/drivers/infiniband/hw/erdma/erdma_eq.c @@ -13,7 +13,7 @@ void notify_eq(struct erdma_eq *eq) u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq->ci) | FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1); - *eq->db_record = db_data; + *eq->dbrec = db_data; writeq(db_data, eq->db); atomic64_inc(&eq->notify_num); @@ -83,14 +83,11 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) int erdma_aeq_init(struct erdma_dev *dev) { struct erdma_eq *eq = &dev->aeq; - u32 buf_size; eq->depth = ERDMA_DEFAULT_EQ_DEPTH; - buf_size = eq->depth << EQE_SHIFT; - eq->qbuf = - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), - &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); + eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, + &eq->qbuf_dma_addr, GFP_KERNEL); if (!eq->qbuf) return -ENOMEM; @@ -99,26 +96,34 @@ int erdma_aeq_init(struct erdma_dev *dev) atomic64_set(&eq->notify_num, 0); eq->db = dev->func_bar + ERDMA_REGS_AEQ_DB_REG; - eq->db_record = (u64 *)(eq->qbuf + buf_size); + eq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &eq->dbrec_dma); + if (!eq->dbrec) + goto err_out; erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG, upper_32_bits(eq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_L_REG, lower_32_bits(eq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth); - erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, - eq->qbuf_dma_addr + buf_size); + erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, eq->dbrec_dma); return 0; + +err_out: + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, + eq->qbuf_dma_addr); + + return -ENOMEM; } void erdma_aeq_destroy(struct erdma_dev *dev) { struct erdma_eq *eq = &dev->aeq; - dma_free_coherent(&dev->pdev->dev, - WARPPED_BUFSIZE(eq->depth << EQE_SHIFT), eq->qbuf, + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, eq->qbuf_dma_addr); + + dma_pool_free(dev->db_pool, eq->dbrec, eq->dbrec_dma); } void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) @@ -209,7 +214,6 @@ static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn) static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) { struct erdma_cmdq_create_eq_req req; - dma_addr_t db_info_dma_addr; erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, CMDQ_OPCODE_CREATE_EQ); @@ -219,9 +223,8 @@ static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) req.qtype = ERDMA_EQ_TYPE_CEQ; /* Vector index is the same as EQN. */ req.vector_idx = eqn; - db_info_dma_addr = eq->qbuf_dma_addr + (eq->depth << EQE_SHIFT); - req.db_dma_addr_l = lower_32_bits(db_info_dma_addr); - req.db_dma_addr_h = upper_32_bits(db_info_dma_addr); + req.db_dma_addr_l = lower_32_bits(eq->dbrec_dma); + req.db_dma_addr_h = upper_32_bits(eq->dbrec_dma); return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); } @@ -229,12 +232,11 @@ static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) { struct erdma_eq *eq = &dev->ceqs[ceqn].eq; - u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT; int ret; - eq->qbuf = - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), - &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); + eq->depth = ERDMA_DEFAULT_EQ_DEPTH; + eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, + &eq->qbuf_dma_addr, GFP_KERNEL); if (!eq->qbuf) return -ENOMEM; @@ -242,10 +244,16 @@ static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) atomic64_set(&eq->event_num, 0); atomic64_set(&eq->notify_num, 0); - eq->depth = ERDMA_DEFAULT_EQ_DEPTH; eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG + (ceqn + 1) * ERDMA_DB_SIZE; - eq->db_record = (u64 *)(eq->qbuf + buf_size); + + eq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &eq->dbrec_dma); + if (!eq->dbrec) { + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, + eq->qbuf, eq->qbuf_dma_addr); + return -ENOMEM; + } + eq->ci = 0; dev->ceqs[ceqn].dev = dev; @@ -259,7 +267,6 @@ static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn) { struct erdma_eq *eq = &dev->ceqs[ceqn].eq; - u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT; struct erdma_cmdq_destroy_eq_req req; int err; @@ -276,8 +283,9 @@ static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn) if (err) return; - dma_free_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), eq->qbuf, + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, eq->qbuf_dma_addr); + dma_pool_free(dev->db_pool, eq->dbrec, eq->dbrec_dma); } int erdma_ceqs_init(struct erdma_dev *dev) diff --git a/drivers/infiniband/hw/erdma/erdma_hw.h b/drivers/infiniband/hw/erdma/erdma_hw.h index 3212a1222760..05978f3b1475 100644 --- a/drivers/infiniband/hw/erdma/erdma_hw.h +++ b/drivers/infiniband/hw/erdma/erdma_hw.h @@ -240,7 +240,7 @@ struct erdma_cmdq_create_cq_req { u32 qbuf_addr_l; u32 qbuf_addr_h; u32 cfg1; - u64 cq_db_info_addr; + u64 cq_dbrec_dma; u32 first_page_offset; u32 cfg2; }; @@ -335,8 +335,8 @@ struct erdma_cmdq_create_qp_req { u64 rq_buf_addr; u32 sq_mtt_cfg; u32 rq_mtt_cfg; - u64 sq_db_info_dma_addr; - u64 rq_db_info_dma_addr; + u64 sq_dbrec_dma; + u64 rq_dbrec_dma; u64 sq_mtt_entry[3]; u64 rq_mtt_entry[3]; diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c index 472939172f0c..7080f8a71ec4 100644 --- a/drivers/infiniband/hw/erdma/erdma_main.c +++ b/drivers/infiniband/hw/erdma/erdma_main.c @@ -178,16 +178,26 @@ static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev) if (!dev->resp_pool) return -ENOMEM; + dev->db_pool = dma_pool_create("erdma_db_pool", &pdev->dev, + ERDMA_DB_SIZE, ERDMA_DB_SIZE, 0); + if (!dev->db_pool) { + ret = -ENOMEM; + goto destroy_resp_pool; + } + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(ERDMA_PCI_WIDTH)); if (ret) - goto destroy_pool; + goto destroy_db_pool; dma_set_max_seg_size(&pdev->dev, UINT_MAX); return 0; -destroy_pool: +destroy_db_pool: + dma_pool_destroy(dev->db_pool); + +destroy_resp_pool: dma_pool_destroy(dev->resp_pool); return ret; @@ -195,6 +205,7 @@ destroy_pool: static void erdma_device_uninit(struct erdma_dev *dev) { + dma_pool_destroy(dev->db_pool); dma_pool_destroy(dev->resp_pool); } diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c index 6d0330badd68..4d1f9114cd97 100644 --- a/drivers/infiniband/hw/erdma/erdma_qp.c +++ b/drivers/infiniband/hw/erdma/erdma_qp.c @@ -492,7 +492,7 @@ static void kick_sq_db(struct erdma_qp *qp, u16 pi) u64 db_data = FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp)) | FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, pi); - *(u64 *)qp->kern_qp.sq_db_info = db_data; + *(u64 *)qp->kern_qp.sq_dbrec = db_data; writeq(db_data, qp->kern_qp.hw_sq_db); } @@ -557,7 +557,7 @@ static int erdma_post_recv_one(struct erdma_qp *qp, return -EINVAL; } - *(u64 *)qp->kern_qp.rq_db_info = *(u64 *)rqe; + *(u64 *)qp->kern_qp.rq_dbrec = *(u64 *)rqe; writeq(*(u64 *)rqe, qp->kern_qp.hw_rq_db); qp->kern_qp.rwr_tbl[qp->kern_qp.rq_pi & (qp->attrs.rq_size - 1)] = diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 23dfc01603f8..40c9b6e46b82 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -76,10 +76,8 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; - req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr + - (qp->attrs.sq_size << SQEBB_SHIFT); - req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr + - (qp->attrs.rq_size << RQE_SHIFT); + req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma; + req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma; } else { user_qp = &qp->user_qp; req.sq_cqn_mtt_cfg = FIELD_PREP( @@ -107,8 +105,8 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg, &req.rq_buf_addr, req.rq_mtt_entry); - req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr; - req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr; + req.sq_dbrec_dma = user_qp->sq_dbrec_dma; + req.rq_dbrec_dma = user_qp->rq_dbrec_dma; if (uctx->ext_db.enable) { req.sq_cqn_mtt_cfg |= @@ -209,8 +207,7 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) ERDMA_MR_MTT_0LEVEL); req.first_page_offset = 0; - req.cq_db_info_addr = - cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT); + req.cq_dbrec_dma = cq->kern_cq.dbrec_dma; } else { mem = &cq->user_cq.qbuf_mem; req.cfg0 |= @@ -233,7 +230,7 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) mem->mtt_nents); req.first_page_offset = mem->page_offset; - req.cq_db_info_addr = cq->user_cq.db_info_dma_addr; + req.cq_dbrec_dma = cq->user_cq.dbrec_dma; if (uctx->ext_db.enable) { req.cfg1 |= FIELD_PREP( @@ -482,16 +479,24 @@ static void free_kernel_qp(struct erdma_qp *qp) vfree(qp->kern_qp.rwr_tbl); if (qp->kern_qp.sq_buf) - dma_free_coherent( - &dev->pdev->dev, - WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), - qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); + dma_free_coherent(&dev->pdev->dev, + qp->attrs.sq_size << SQEBB_SHIFT, + qp->kern_qp.sq_buf, + qp->kern_qp.sq_buf_dma_addr); + + if (qp->kern_qp.sq_dbrec) + dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, + qp->kern_qp.sq_dbrec_dma); if (qp->kern_qp.rq_buf) - dma_free_coherent( - &dev->pdev->dev, - WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), - qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); + dma_free_coherent(&dev->pdev->dev, + qp->attrs.rq_size << RQE_SHIFT, + qp->kern_qp.rq_buf, + qp->kern_qp.rq_buf_dma_addr); + + if (qp->kern_qp.rq_dbrec) + dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, + qp->kern_qp.rq_dbrec_dma); } static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, @@ -516,20 +521,27 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, if (!kqp->swr_tbl || !kqp->rwr_tbl) goto err_out; - size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; + size = qp->attrs.sq_size << SQEBB_SHIFT; kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, &kqp->sq_buf_dma_addr, GFP_KERNEL); if (!kqp->sq_buf) goto err_out; - size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; + kqp->sq_dbrec = + dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma); + if (!kqp->sq_dbrec) + goto err_out; + + size = qp->attrs.rq_size << RQE_SHIFT; kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, &kqp->rq_buf_dma_addr, GFP_KERNEL); if (!kqp->rq_buf) goto err_out; - kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT); - kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT); + kqp->rq_dbrec = + dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma); + if (!kqp->rq_dbrec) + goto err_out; return 0; @@ -864,9 +876,9 @@ erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx, } static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, - u64 va, u32 len, u64 db_info_va) + u64 va, u32 len, u64 dbrec_va) { - dma_addr_t db_info_dma_addr; + dma_addr_t dbrec_dma; u32 rq_offset; int ret; @@ -889,14 +901,14 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, if (ret) goto put_sq_mtt; - ret = erdma_map_user_dbrecords(uctx, db_info_va, + ret = erdma_map_user_dbrecords(uctx, dbrec_va, &qp->user_qp.user_dbr_page, - &db_info_dma_addr); + &dbrec_dma); if (ret) goto put_rq_mtt; - qp->user_qp.sq_db_info_dma_addr = db_info_dma_addr; - qp->user_qp.rq_db_info_dma_addr = db_info_dma_addr + ERDMA_DB_SIZE; + qp->user_qp.sq_dbrec_dma = dbrec_dma; + qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE; return 0; @@ -1237,9 +1249,10 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) return err; if (rdma_is_kernel_res(&cq->ibcq.res)) { - dma_free_coherent(&dev->pdev->dev, - WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), + dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, + cq->kern_cq.dbrec_dma); } else { erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); put_mtt_entries(dev, &cq->user_cq.qbuf_mem); @@ -1279,16 +1292,7 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) wait_for_completion(&qp->safe_free); if (rdma_is_kernel_res(&qp->ibqp.res)) { - vfree(qp->kern_qp.swr_tbl); - vfree(qp->kern_qp.rwr_tbl); - dma_free_coherent( - &dev->pdev->dev, - WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), - qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); - dma_free_coherent( - &dev->pdev->dev, - WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), - qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); + free_kernel_qp(qp); } else { put_mtt_entries(dev, &qp->user_qp.sq_mem); put_mtt_entries(dev, &qp->user_qp.rq_mem); @@ -1588,7 +1592,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va, &cq->user_cq.user_dbr_page, - &cq->user_cq.db_info_dma_addr); + &cq->user_cq.dbrec_dma); if (ret) put_mtt_entries(dev, &cq->user_cq.qbuf_mem); @@ -1600,19 +1604,27 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq) struct erdma_dev *dev = to_edev(cq->ibcq.device); cq->kern_cq.qbuf = - dma_alloc_coherent(&dev->pdev->dev, - WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), + dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL); if (!cq->kern_cq.qbuf) return -ENOMEM; - cq->kern_cq.db_record = - (u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT)); + cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, + &cq->kern_cq.dbrec_dma); + if (!cq->kern_cq.dbrec) + goto err_out; + spin_lock_init(&cq->kern_cq.lock); /* use default cqdb addr */ cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET; return 0; + +err_out: + dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, + cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + + return -ENOMEM; } int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, @@ -1676,9 +1688,10 @@ err_free_res: erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); put_mtt_entries(dev, &cq->user_cq.qbuf_mem); } else { - dma_free_coherent(&dev->pdev->dev, - WARPPED_BUFSIZE(depth << CQE_SHIFT), + dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT, cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, + cq->kern_cq.dbrec_dma); } err_out_xa: diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index db6018529ccc..4f02ba06b210 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -140,8 +140,8 @@ struct erdma_uqp { struct erdma_mem sq_mem; struct erdma_mem rq_mem; - dma_addr_t sq_db_info_dma_addr; - dma_addr_t rq_db_info_dma_addr; + dma_addr_t sq_dbrec_dma; + dma_addr_t rq_dbrec_dma; struct erdma_user_dbrecords_page *user_dbr_page; @@ -167,8 +167,11 @@ struct erdma_kqp { void *rq_buf; dma_addr_t rq_buf_dma_addr; - void *sq_db_info; - void *rq_db_info; + void *sq_dbrec; + void *rq_dbrec; + + dma_addr_t sq_dbrec_dma; + dma_addr_t rq_dbrec_dma; u8 sig_all; }; @@ -246,13 +249,14 @@ struct erdma_kcq_info { spinlock_t lock; u8 __iomem *db; - u64 *db_record; + u64 *dbrec; + dma_addr_t dbrec_dma; }; struct erdma_ucq_info { struct erdma_mem qbuf_mem; struct erdma_user_dbrecords_page *user_dbr_page; - dma_addr_t db_info_dma_addr; + dma_addr_t dbrec_dma; }; struct erdma_cq { diff --git a/drivers/infiniband/hw/hfi1/ipoib_main.c b/drivers/infiniband/hw/hfi1/ipoib_main.c index 5d814afdf7f3..7c9d5203002b 100644 --- a/drivers/infiniband/hw/hfi1/ipoib_main.c +++ b/drivers/infiniband/hw/hfi1/ipoib_main.c @@ -21,36 +21,25 @@ static int hfi1_ipoib_dev_init(struct net_device *dev) struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); int ret; - dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); - if (!dev->tstats) - return -ENOMEM; - ret = priv->netdev_ops->ndo_init(dev); if (ret) - goto out_ret; + return ret; ret = hfi1_netdev_add_data(priv->dd, qpn_from_mac(priv->netdev->dev_addr), dev); if (ret < 0) { priv->netdev_ops->ndo_uninit(dev); - goto out_ret; + return ret; } return 0; -out_ret: - free_percpu(dev->tstats); - dev->tstats = NULL; - return ret; } static void hfi1_ipoib_dev_uninit(struct net_device *dev) { struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); - free_percpu(dev->tstats); - dev->tstats = NULL; - hfi1_netdev_remove_data(priv->dd, qpn_from_mac(priv->netdev->dev_addr)); priv->netdev_ops->ndo_uninit(dev); @@ -107,7 +96,6 @@ static const struct net_device_ops hfi1_ipoib_netdev_ops = { .ndo_uninit = hfi1_ipoib_dev_uninit, .ndo_open = hfi1_ipoib_dev_open, .ndo_stop = hfi1_ipoib_dev_stop, - .ndo_get_stats64 = dev_get_tstats64, }; static int hfi1_ipoib_mcast_attach(struct net_device *dev, @@ -173,9 +161,6 @@ static void hfi1_ipoib_netdev_dtor(struct net_device *dev) hfi1_ipoib_txreq_deinit(priv); hfi1_ipoib_rxq_deinit(priv->netdev); - - free_percpu(dev->tstats); - dev->tstats = NULL; } static void hfi1_ipoib_set_id(struct net_device *dev, int id) @@ -234,6 +219,7 @@ static int hfi1_ipoib_setup_rn(struct ib_device *device, netdev->priv_destructor = hfi1_ipoib_netdev_dtor; netdev->needs_free_netdev = true; + netdev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; return 0; } diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c index 119ec2f1382b..7133964749f8 100644 --- a/drivers/infiniband/hw/hfi1/pcie.c +++ b/drivers/infiniband/hw/hfi1/pcie.c @@ -1207,14 +1207,11 @@ retry: (u32)lnkctl2); /* only write to parent if target is not as high as ours */ if ((lnkctl2 & PCI_EXP_LNKCTL2_TLS) < target_vector) { - lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS; - lnkctl2 |= target_vector; - dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__, - (u32)lnkctl2); - ret = pcie_capability_write_word(parent, - PCI_EXP_LNKCTL2, lnkctl2); + ret = pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL2, + PCI_EXP_LNKCTL2_TLS, + target_vector); if (ret) { - dd_dev_err(dd, "Unable to write to PCI config\n"); + dd_dev_err(dd, "Unable to change parent PCI target speed\n"); return_error = 1; goto done; } @@ -1223,22 +1220,11 @@ retry: } dd_dev_info(dd, "%s: setting target link speed\n", __func__); - ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL2, &lnkctl2); + ret = pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL2, + PCI_EXP_LNKCTL2_TLS, + target_vector); if (ret) { - dd_dev_err(dd, "Unable to read from PCI config\n"); - return_error = 1; - goto done; - } - - dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__, - (u32)lnkctl2); - lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS; - lnkctl2 |= target_vector; - dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__, - (u32)lnkctl2); - ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL2, lnkctl2); - if (ret) { - dd_dev_err(dd, "Unable to write to PCI config\n"); + dd_dev_err(dd, "Unable to change device PCI target speed\n"); return_error = 1; goto done; } diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index b4209b6aed8d..3e02c474f59f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -59,8 +59,10 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct hns_roce_dev *hr_dev = to_hr_dev(ibah->device); struct hns_roce_ib_create_ah_resp resp = {}; struct hns_roce_ah *ah = to_hr_ah(ibah); - int ret = 0; - u32 max_sl; + u8 tclass = get_tclass(grh); + u8 priority = 0; + u8 tc_mode = 0; + int ret; if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && udata) return -EOPNOTSUPP; @@ -74,16 +76,23 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, ah->av.hop_limit = grh->hop_limit; ah->av.flowlabel = grh->flow_label; ah->av.udp_sport = get_ah_udp_sport(ah_attr); - ah->av.tclass = get_tclass(grh); - - ah->av.sl = rdma_ah_get_sl(ah_attr); - max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev->caps.sl_num - 1); - if (unlikely(ah->av.sl > max_sl)) { - ibdev_err_ratelimited(&hr_dev->ib_dev, - "failed to set sl, sl (%u) shouldn't be larger than %u.\n", - ah->av.sl, max_sl); + ah->av.tclass = tclass; + + ret = hr_dev->hw->get_dscp(hr_dev, tclass, &tc_mode, &priority); + if (ret == -EOPNOTSUPP) + ret = 0; + + if (ret && grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) + return ret; + + if (tc_mode == HNAE3_TC_MAP_MODE_DSCP && + grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) + ah->av.sl = priority; + else + ah->av.sl = rdma_ah_get_sl(ah_attr); + + if (!check_sl_valid(hr_dev, ah->av.sl)) return -EINVAL; - } memcpy(ah->av.dgid, grh->dgid.raw, HNS_ROCE_GID_SIZE); memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN); @@ -99,6 +108,8 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, } if (udata) { + resp.priority = ah->av.sl; + resp.tc_mode = tc_mode; memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN); ret = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index 11a78ceae568..950c133d4220 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -153,8 +153,7 @@ int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, return total; } -int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, - int buf_cnt, struct ib_umem *umem, +int hns_roce_get_umem_bufs(dma_addr_t *bufs, int buf_cnt, struct ib_umem *umem, unsigned int page_shift) { struct ib_block_iter biter; diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 7250d0643b5c..56dc3908da2f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -149,7 +149,7 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) return ret; } - ret = xa_err(xa_store(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL)); + ret = xa_err(xa_store_irq(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL)); if (ret) { ibdev_err(ibdev, "failed to xa_store CQ, ret = %d.\n", ret); goto err_put; @@ -163,7 +163,7 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) return 0; err_xa: - xa_erase(&cq_table->array, hr_cq->cqn); + xa_erase_irq(&cq_table->array, hr_cq->cqn); err_put: hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); @@ -182,7 +182,7 @@ static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) dev_err(dev, "DESTROY_CQ failed (%d) for CQN %06lx\n", ret, hr_cq->cqn); - xa_erase(&cq_table->array, hr_cq->cqn); + xa_erase_irq(&cq_table->array, hr_cq->cqn); /* Waiting interrupt process procedure carried out */ synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq); @@ -476,13 +476,6 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) struct ib_event event; struct ib_cq *ibcq; - hr_cq = xa_load(&hr_dev->cq_table.array, - cqn & (hr_dev->caps.num_cqs - 1)); - if (!hr_cq) { - dev_warn(dev, "async event for bogus CQ 0x%06x\n", cqn); - return; - } - if (event_type != HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID && event_type != HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR && event_type != HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW) { @@ -491,7 +484,16 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) return; } - refcount_inc(&hr_cq->refcount); + xa_lock(&hr_dev->cq_table.array); + hr_cq = xa_load(&hr_dev->cq_table.array, + cqn & (hr_dev->caps.num_cqs - 1)); + if (hr_cq) + refcount_inc(&hr_cq->refcount); + xa_unlock(&hr_dev->cq_table.array); + if (!hr_cq) { + dev_warn(dev, "async event for bogus CQ 0x%06x\n", cqn); + return; + } ibcq = &hr_cq->ib_cq; if (ibcq->event_handler) { @@ -534,4 +536,5 @@ void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev) for (i = 0; i < HNS_ROCE_CQ_BANK_NUM; i++) ida_destroy(&hr_dev->cq_table.bank[i].ida); + mutex_destroy(&hr_dev->cq_table.bank_mutex); } diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index c3cbd0a494bf..ff0b3f68ee3a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -100,6 +100,9 @@ #define CQ_BANKID_SHIFT 2 #define CQ_BANKID_MASK GENMASK(1, 0) +#define HNS_ROCE_MAX_CQ_COUNT 0xFFFF +#define HNS_ROCE_MAX_CQ_PERIOD 0xFFFF + enum { SERV_TYPE_RC, SERV_TYPE_UC, @@ -645,6 +648,8 @@ struct hns_roce_qp { struct hns_user_mmap_entry *dwqe_mmap_entry; u32 config; enum hns_roce_cong_type cong_type; + u8 tc_mode; + u8 priority; }; struct hns_roce_ib_iboe { @@ -923,8 +928,7 @@ struct hns_roce_hw { int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, int flags, void *mb_buf); - int (*frmr_write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf, - struct hns_roce_mr *mr); + int (*frmr_write_mtpt)(void *mb_buf, struct hns_roce_mr *mr); int (*mw_write_mtpt)(void *mb_buf, struct hns_roce_mw *mw); void (*write_cqc)(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts, @@ -950,6 +954,8 @@ struct hns_roce_hw { int (*query_sccc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer); int (*query_hw_counter)(struct hns_roce_dev *hr_dev, u64 *stats, u32 port, int *hw_counters); + int (*get_dscp)(struct hns_roce_dev *hr_dev, u8 dscp, + u8 *tc_mode, u8 *priority); const struct ib_device_ops *hns_roce_dev_ops; const struct ib_device_ops *hns_roce_dev_srq_ops; }; @@ -1228,7 +1234,7 @@ struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, int buf_cnt, struct hns_roce_buf *buf, unsigned int page_shift); -int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, +int hns_roce_get_umem_bufs(dma_addr_t *bufs, int buf_cnt, struct ib_umem *umem, unsigned int page_shift); @@ -1292,4 +1298,6 @@ struct hns_user_mmap_entry * hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, size_t length, enum hns_roce_mmap_type mmap_type); +bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl); + #endif /* _HNS_ROCE_DEVICE_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index a4b3f19161dc..02baa853a76c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -281,7 +281,7 @@ static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, return hem; fail: - hns_roce_free_hem(hr_dev, hem); + kfree(hem); return NULL; } @@ -877,6 +877,7 @@ void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, if (hns_roce_check_whether_mhop(hr_dev, table->type)) { hns_roce_cleanup_mhop_hem_table(hr_dev, table); + mutex_destroy(&table->mutex); return; } @@ -891,6 +892,7 @@ void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, hns_roce_free_hem(hr_dev, table->hem[i]); } + mutex_destroy(&table->mutex); kfree(table->hem); } @@ -986,15 +988,13 @@ static void hem_list_free_all(struct hns_roce_dev *hr_dev, } } -static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, - u64 table_addr) +static void hem_list_link_bt(void *base_addr, u64 table_addr) { *(u64 *)(base_addr) = table_addr; } /* assign L0 table address to hem from root bt */ -static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_item *hem, void *cpu_addr, +static void hem_list_assign_bt(struct hns_roce_hem_item *hem, void *cpu_addr, u64 phy_addr) { hem->addr = cpu_addr; @@ -1163,8 +1163,7 @@ static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, if (level > 1) { pre = hem_ptrs[level - 1]; step = (cur->start - pre->start) / step * BA_BYTE_LEN; - hem_list_link_bt(hr_dev, pre->addr + step, - cur->dma_addr); + hem_list_link_bt(pre->addr + step, cur->dma_addr); } } @@ -1222,7 +1221,7 @@ static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base, if (!hem) return -ENOMEM; - hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); + hem_list_assign_bt(hem, cpu_base, phy_base); list_add(&hem->list, branch_head); list_add(&hem->sibling, leaf_head); @@ -1245,7 +1244,7 @@ static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base, /* if exist mid bt, link L1 to L0 */ list_for_each_entry_safe(hem, temp_hem, branch_head, list) { offset = (hem->start - r->offset) / step * BA_BYTE_LEN; - hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr); + hem_list_link_bt(cpu_base + offset, hem->dma_addr); total++; } diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h index 6fb51db9682b..9c415b2541af 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.h +++ b/drivers/infiniband/hw/hns/hns_roce_hem.h @@ -57,16 +57,16 @@ enum { }; #define check_whether_bt_num_3(type, hop_num) \ - (type < HEM_TYPE_MTT && hop_num == 2) + ((type) < HEM_TYPE_MTT && (hop_num) == 2) #define check_whether_bt_num_2(type, hop_num) \ - ((type < HEM_TYPE_MTT && hop_num == 1) || \ - (type >= HEM_TYPE_MTT && hop_num == 2)) + (((type) < HEM_TYPE_MTT && (hop_num) == 1) || \ + ((type) >= HEM_TYPE_MTT && (hop_num) == 2)) #define check_whether_bt_num_1(type, hop_num) \ - ((type < HEM_TYPE_MTT && hop_num == HNS_ROCE_HOP_NUM_0) || \ - (type >= HEM_TYPE_MTT && hop_num == 1) || \ - (type >= HEM_TYPE_MTT && hop_num == HNS_ROCE_HOP_NUM_0)) + (((type) < HEM_TYPE_MTT && (hop_num) == HNS_ROCE_HOP_NUM_0) || \ + ((type) >= HEM_TYPE_MTT && (hop_num) == 1) || \ + ((type) >= HEM_TYPE_MTT && (hop_num) == HNS_ROCE_HOP_NUM_0)) struct hns_roce_hem { void *buf; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index ba7ae792d279..4287818a737f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -443,10 +443,6 @@ static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel); - - if (WARN_ON(ah->av.sl > MAX_SERVICE_LEVEL)) - return -EINVAL; - hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl); ud_sq_wqe->sgid_index = ah->av.gid_index; @@ -2105,7 +2101,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz)); - caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE / + caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz); } else { u32 func_num = max_t(u32, 1, hr_dev->func_num); @@ -2671,6 +2667,8 @@ static void free_mr_exit(struct hns_roce_dev *hr_dev) kfree(free_mr->rsv_pd); free_mr->rsv_pd = NULL; } + + mutex_destroy(&free_mr->mutex); } static int free_mr_alloc_res(struct hns_roce_dev *hr_dev) @@ -2821,8 +2819,10 @@ static int free_mr_init(struct hns_roce_dev *hr_dev) mutex_init(&free_mr->mutex); ret = free_mr_alloc_res(hr_dev); - if (ret) + if (ret) { + mutex_destroy(&free_mr->mutex); return ret; + } ret = free_mr_modify_qp(hr_dev); if (ret) @@ -3208,13 +3208,14 @@ static int set_mtpt_pbl(struct hns_roce_dev *hr_dev, /* Aligned to the hardware address access unit */ for (i = 0; i < ARRAY_SIZE(pages); i++) - pages[i] >>= 6; + pages[i] >>= MPT_PBL_BUF_ADDR_S; pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr); mpt_entry->pbl_size = cpu_to_le32(mr->npages); - mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3); - hr_reg_write(mpt_entry, MPT_PBL_BA_H, upper_32_bits(pbl_ba >> 3)); + mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> MPT_PBL_BA_ADDR_S); + hr_reg_write(mpt_entry, MPT_PBL_BA_H, + upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S)); mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0])); hr_reg_write(mpt_entry, MPT_PA0_H, upper_32_bits(pages[0])); @@ -3307,8 +3308,7 @@ static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev, return ret; } -static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev, - void *mb_buf, struct hns_roce_mr *mr) +static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr) { dma_addr_t pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr); struct hns_roce_v2_mpt_entry *mpt_entry; @@ -3335,8 +3335,10 @@ static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev, mpt_entry->pbl_size = cpu_to_le32(mr->npages); - mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3)); - hr_reg_write(mpt_entry, MPT_PBL_BA_H, upper_32_bits(pbl_ba >> 3)); + mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> + MPT_PBL_BA_ADDR_S)); + hr_reg_write(mpt_entry, MPT_PBL_BA_H, + upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S)); return 0; } @@ -3582,14 +3584,14 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift)); hr_reg_write(cq_context, CQC_CQE_BUF_PG_SZ, to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift)); - hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle >> 3); - hr_reg_write(cq_context, CQC_CQE_BA_H, (dma_handle >> (32 + 3))); + hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle >> CQC_CQE_BA_L_S); + hr_reg_write(cq_context, CQC_CQE_BA_H, dma_handle >> CQC_CQE_BA_H_S); hr_reg_write_bool(cq_context, CQC_DB_RECORD_EN, hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB); hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_L, ((u32)hr_cq->db.dma) >> 1); hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_H, - hr_cq->db.dma >> 32); + hr_cq->db.dma >> CQC_CQE_DB_RECORD_ADDR_H_S); hr_reg_write(cq_context, CQC_CQ_MAX_CNT, HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM); hr_reg_write(cq_context, CQC_CQ_PERIOD, @@ -3711,8 +3713,9 @@ static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, wc->status == IB_WC_WR_FLUSH_ERR)) return; - ibdev_err(&hr_dev->ib_dev, "error cqe status 0x%x:\n", cqe_status); - print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe, + ibdev_err_ratelimited(&hr_dev->ib_dev, "error cqe status 0x%x:\n", + cqe_status); + print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_NONE, 16, 4, cqe, cq->cqe_size, false); wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS); @@ -4217,8 +4220,7 @@ static void set_access_flags(struct hns_roce_qp *hr_qp, } static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp, - struct hns_roce_v2_qp_context *context, - struct hns_roce_v2_qp_context *qpc_mask) + struct hns_roce_v2_qp_context *context) { hr_reg_write(context, QPC_SGE_SHIFT, to_hr_hem_entries_shift(hr_qp->sge.sge_cnt, @@ -4240,7 +4242,6 @@ static inline int get_pdn(struct ib_pd *ib_pd) } static void modify_qp_reset_to_init(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, struct hns_roce_v2_qp_context *context, struct hns_roce_v2_qp_context *qpc_mask) { @@ -4259,7 +4260,7 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, hr_reg_write(context, QPC_RQWS, ilog2(hr_qp->rq.max_gs)); - set_qpc_wqe_cnt(hr_qp, context, qpc_mask); + set_qpc_wqe_cnt(hr_qp, context); /* No VLAN need to set 0xFFF */ hr_reg_write(context, QPC_VLAN_ID, 0xfff); @@ -4300,7 +4301,6 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, } static void modify_qp_init_to_init(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, struct hns_roce_v2_qp_context *context, struct hns_roce_v2_qp_context *qpc_mask) { @@ -4521,16 +4521,16 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, return -EINVAL; } - hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba >> 4); + hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba >> QPC_TRRL_BA_L_S); hr_reg_clear(qpc_mask, QPC_TRRL_BA_L); - context->trrl_ba = cpu_to_le32(trrl_ba >> (16 + 4)); + context->trrl_ba = cpu_to_le32(trrl_ba >> QPC_TRRL_BA_M_S); qpc_mask->trrl_ba = 0; - hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba >> (32 + 16 + 4)); + hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba >> QPC_TRRL_BA_H_S); hr_reg_clear(qpc_mask, QPC_TRRL_BA_H); - context->irrl_ba = cpu_to_le32(irrl_ba >> 6); + context->irrl_ba = cpu_to_le32(irrl_ba >> QPC_IRRL_BA_L_S); qpc_mask->irrl_ba = 0; - hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba >> (32 + 6)); + hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba >> QPC_IRRL_BA_H_S); hr_reg_clear(qpc_mask, QPC_IRRL_BA_H); hr_reg_enable(context, QPC_RMT_E2E); @@ -4592,8 +4592,9 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, hr_reg_clear(qpc_mask, QPC_TRRL_HEAD_MAX); hr_reg_clear(qpc_mask, QPC_TRRL_TAIL_MAX); +#define MAX_LP_SGEN 3 /* rocee send 2^lp_sgen_ini segs every time */ - hr_reg_write(context, QPC_LP_SGEN_INI, 3); + hr_reg_write(context, QPC_LP_SGEN_INI, MAX_LP_SGEN); hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI); if (udata && ibqp->qp_type == IB_QPT_RC && @@ -4619,8 +4620,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, return 0; } -static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, int attr_mask, +static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, int attr_mask, struct hns_roce_v2_qp_context *context, struct hns_roce_v2_qp_context *qpc_mask) { @@ -4685,7 +4685,7 @@ static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr, *tail = (*tail == hr_dev->caps.num_qps - 1) ? 0 : (*tail + 1); list_for_each_entry(hr_dip, &hr_dev->dip_list, node) { - if (!memcmp(grh->dgid.raw, hr_dip->dgid, 16)) { + if (!memcmp(grh->dgid.raw, hr_dip->dgid, GID_LEN_V2)) { *dip_idx = hr_dip->dip_idx; goto out; } @@ -4828,6 +4828,69 @@ static int fill_cong_field(struct ib_qp *ibqp, const struct ib_qp_attr *attr, return 0; } +static int hns_roce_hw_v2_get_dscp(struct hns_roce_dev *hr_dev, u8 dscp, + u8 *tc_mode, u8 *priority) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (!ops->get_dscp_prio) + return -EOPNOTSUPP; + + return ops->get_dscp_prio(handle, dscp, tc_mode, priority); +} + +bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl) +{ + u32 max_sl; + + max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev->caps.sl_num - 1); + if (unlikely(sl > max_sl)) { + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to set SL(%u). Shouldn't be larger than %u.\n", + sl, max_sl); + return false; + } + + return true; +} + +static int hns_roce_set_sl(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct ib_device *ibdev = &hr_dev->ib_dev; + int ret; + + ret = hns_roce_hw_v2_get_dscp(hr_dev, get_tclass(&attr->ah_attr.grh), + &hr_qp->tc_mode, &hr_qp->priority); + if (ret && ret != -EOPNOTSUPP && + grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + ibdev_err_ratelimited(ibdev, + "failed to get dscp, ret = %d.\n", ret); + return ret; + } + + if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP && + grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) + hr_qp->sl = hr_qp->priority; + else + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + + if (!check_sl_valid(hr_dev, hr_qp->sl)) + return -EINVAL; + + hr_reg_write(context, QPC_SL, hr_qp->sl); + hr_reg_clear(qpc_mask, QPC_SL); + + return 0; +} + static int hns_roce_v2_set_path(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, @@ -4843,25 +4906,18 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, int is_roce_protocol; u16 vlan_id = 0xffff; bool is_udp = false; - u32 max_sl; u8 ib_port; u8 hr_port; int ret; - max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev->caps.sl_num - 1); - if (unlikely(sl > max_sl)) { - ibdev_err_ratelimited(ibdev, - "failed to fill QPC, sl (%u) shouldn't be larger than %u.\n", - sl, max_sl); - return -EINVAL; - } - /* * If free_mr_en of qp is set, it means that this qp comes from * free mr. This qp will perform the loopback operation. * In the loopback scenario, only sl needs to be set. */ if (hr_qp->free_mr_en) { + if (!check_sl_valid(hr_dev, sl)) + return -EINVAL; hr_reg_write(context, QPC_SL, sl); hr_reg_clear(qpc_mask, QPC_SL); hr_qp->sl = sl; @@ -4931,11 +4987,7 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw)); - hr_qp->sl = sl; - hr_reg_write(context, QPC_SL, hr_qp->sl); - hr_reg_clear(qpc_mask, QPC_SL); - - return 0; + return hns_roce_set_sl(ibqp, attr, context, qpc_mask); } static bool check_qp_state(enum ib_qp_state cur_state, @@ -4982,15 +5034,14 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { memset(qpc_mask, 0, hr_dev->caps.qpc_sz); - modify_qp_reset_to_init(ibqp, attr, context, qpc_mask); + modify_qp_reset_to_init(ibqp, context, qpc_mask); } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { - modify_qp_init_to_init(ibqp, attr, context, qpc_mask); + modify_qp_init_to_init(ibqp, context, qpc_mask); } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context, qpc_mask, udata); } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { - ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context, - qpc_mask); + ret = modify_qp_rtr_to_rts(ibqp, attr_mask, context, qpc_mask); } return ret; @@ -5802,7 +5853,7 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) dev_info(hr_dev->dev, "cq_period(%u) reached the upper limit, adjusted to 65.\n", cq_period); - cq_period = HNS_ROCE_MAX_CQ_PERIOD; + cq_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08; } cq_period *= HNS_ROCE_CLOCK_ADJUST; } @@ -6735,6 +6786,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .query_srqc = hns_roce_v2_query_srqc, .query_sccc = hns_roce_v2_query_sccc, .query_hw_counter = hns_roce_hw_v2_query_counter, + .get_dscp = hns_roce_hw_v2_get_dscp, .hns_roce_dev_ops = &hns_roce_v2_dev_ops, .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, }; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index df04bc8ede57..def1d15a03c7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -276,6 +276,10 @@ struct hns_roce_v2_cq_context { __le32 byte_64_se_cqe_idx; }; +#define CQC_CQE_BA_L_S 3 +#define CQC_CQE_BA_H_S (32 + CQC_CQE_BA_L_S) +#define CQC_CQE_DB_RECORD_ADDR_H_S 32 + #define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0 #define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL 0x0 @@ -447,6 +451,12 @@ struct hns_roce_v2_qp_context { struct hns_roce_v2_qp_context_ex ext; }; +#define QPC_TRRL_BA_L_S 4 +#define QPC_TRRL_BA_M_S (16 + QPC_TRRL_BA_L_S) +#define QPC_TRRL_BA_H_S (32 + QPC_TRRL_BA_M_S) +#define QPC_IRRL_BA_L_S 6 +#define QPC_IRRL_BA_H_S (32 + QPC_IRRL_BA_L_S) + #define QPC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_qp_context, h, l) #define QPC_TST QPC_FIELD_LOC(2, 0) @@ -716,6 +726,9 @@ struct hns_roce_v2_mpt_entry { __le32 byte_64_buf_pa1; }; +#define MPT_PBL_BUF_ADDR_S 6 +#define MPT_PBL_BA_ADDR_S 3 + #define MPT_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_mpt_entry, h, l) #define MPT_ST MPT_FIELD_LOC(1, 0) @@ -1334,7 +1347,7 @@ struct fmea_ram_ecc { /* only for RNR timeout issue of HIP08 */ #define HNS_ROCE_CLOCK_ADJUST 1000 -#define HNS_ROCE_MAX_CQ_PERIOD 65 +#define HNS_ROCE_MAX_CQ_PERIOD_HIP08 65 #define HNS_ROCE_MAX_EQ_PERIOD 65 #define HNS_ROCE_RNR_TIMER_10NS 1 #define HNS_ROCE_1US_CFG 999 diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 1dc60c2b2b7a..4cb0af733587 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -37,9 +37,11 @@ #include <rdma/ib_smi.h> #include <rdma/ib_user_verbs.h> #include <rdma/ib_cache.h> +#include "hnae3.h" #include "hns_roce_common.h" #include "hns_roce_device.h" #include "hns_roce_hem.h" +#include "hns_roce_hw_v2.h" static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u32 port, const u8 *addr) @@ -192,6 +194,12 @@ static int hns_roce_query_device(struct ib_device *ib_dev, IB_ATOMIC_HCA : IB_ATOMIC_NONE; props->max_pkeys = 1; props->local_ca_ack_delay = hr_dev->caps.local_ca_ack_delay; + props->max_ah = INT_MAX; + props->cq_caps.max_cq_moderation_period = HNS_ROCE_MAX_CQ_PERIOD; + props->cq_caps.max_cq_moderation_count = HNS_ROCE_MAX_CQ_COUNT; + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) + props->cq_caps.max_cq_moderation_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08; + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { props->max_srq = hr_dev->caps.num_srqs; props->max_srq_wr = hr_dev->caps.max_srq_wrs; @@ -421,6 +429,9 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, return 0; error_fail_copy_to_udata: + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || + hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) + mutex_destroy(&context->page_mutex); hns_roce_dealloc_uar_entry(context); error_fail_uar_entry: @@ -437,6 +448,10 @@ static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext) struct hns_roce_ucontext *context = to_hr_ucontext(ibcontext); struct hns_roce_dev *hr_dev = to_hr_dev(ibcontext->device); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || + hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) + mutex_destroy(&context->page_mutex); + hns_roce_dealloc_uar_entry(context); ida_free(&hr_dev->uar_ida.ida, (int)context->uar.logic_idx); @@ -925,6 +940,15 @@ err_unmap_dmpt: return ret; } +static void hns_roce_teardown_hca(struct hns_roce_dev *hr_dev) +{ + hns_roce_cleanup_bitmap(hr_dev); + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || + hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) + mutex_destroy(&hr_dev->pgdir_mutex); +} + /** * hns_roce_setup_hca - setup host channel adapter * @hr_dev: pointer to hns roce device @@ -973,6 +997,10 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) err_uar_table_free: ida_destroy(&hr_dev->uar_ida.ida); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || + hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) + mutex_destroy(&hr_dev->pgdir_mutex); + return ret; } @@ -1118,7 +1146,7 @@ error_failed_register_device: hr_dev->hw->hw_exit(hr_dev); error_failed_engine_init: - hns_roce_cleanup_bitmap(hr_dev); + hns_roce_teardown_hca(hr_dev); error_failed_setup_hca: hns_roce_cleanup_hem(hr_dev); @@ -1148,7 +1176,7 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev) if (hr_dev->hw->hw_exit) hr_dev->hw->hw_exit(hr_dev); - hns_roce_cleanup_bitmap(hr_dev); + hns_roce_teardown_hca(hr_dev); hns_roce_cleanup_hem(hr_dev); if (hr_dev->cmd_mod) diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 9e05b57a2d67..1a61dceb3319 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -162,7 +162,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, if (mr->type != MR_TYPE_FRMR) ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr); else - ret = hr_dev->hw->frmr_write_mtpt(hr_dev, mailbox->buf, mr); + ret = hr_dev->hw->frmr_write_mtpt(mailbox->buf, mr); if (ret) { dev_err(dev, "failed to write mtpt, ret = %d.\n", ret); goto err_page; @@ -441,18 +441,18 @@ int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_mr *mr = to_hr_mr(ibmr); struct hns_roce_mtr *mtr = &mr->pbl_mtr; - int ret = 0; + int ret, sg_num = 0; mr->npages = 0; mr->page_list = kvcalloc(mr->pbl_mtr.hem_cfg.buf_pg_count, sizeof(dma_addr_t), GFP_KERNEL); if (!mr->page_list) - return ret; + return sg_num; - ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page); - if (ret < 1) { + sg_num = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page); + if (sg_num < 1) { ibdev_err(ibdev, "failed to store sg pages %u %u, cnt = %d.\n", - mr->npages, mr->pbl_mtr.hem_cfg.buf_pg_count, ret); + mr->npages, mr->pbl_mtr.hem_cfg.buf_pg_count, sg_num); goto err_page_list; } @@ -463,17 +463,16 @@ int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, ret = hns_roce_mtr_map(hr_dev, mtr, mr->page_list, mr->npages); if (ret) { ibdev_err(ibdev, "failed to map sg mtr, ret = %d.\n", ret); - ret = 0; + sg_num = 0; } else { mr->pbl_mtr.hem_cfg.buf_pg_shift = (u32)ilog2(ibmr->page_size); - ret = mr->npages; } err_page_list: kvfree(mr->page_list); mr->page_list = NULL; - return ret; + return sg_num; } static void hns_roce_mw_free(struct hns_roce_dev *hr_dev, @@ -756,7 +755,7 @@ static int mtr_map_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) return -ENOMEM; if (mtr->umem) - npage = hns_roce_get_umem_bufs(hr_dev, pages, page_count, + npage = hns_roce_get_umem_bufs(pages, page_count, mtr->umem, page_shift); else npage = hns_roce_get_kmem_bufs(hr_dev, pages, page_count, diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index f35a66325d9a..db34665d1dfb 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -410,7 +410,8 @@ static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) bankid = get_qp_bankid(hr_qp->qpn); - ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3); + ida_free(&hr_dev->qp_table.bank[bankid].ida, + hr_qp->qpn / HNS_ROCE_QP_BANK_NUM); mutex_lock(&hr_dev->qp_table.bank_mutex); hr_dev->qp_table.bank[bankid].inuse--; @@ -1117,7 +1118,6 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, } static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, - struct ib_pd *ib_pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata, struct hns_roce_qp *hr_qp) @@ -1140,7 +1140,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd); if (ret) { ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret); - return ret; + goto err_out; } if (!udata) { @@ -1148,7 +1148,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (ret) { ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n", ret); - return ret; + goto err_out; } } @@ -1219,6 +1219,8 @@ err_qpn: free_qp_buf(hr_dev, hr_qp); err_buf: free_kernel_wrid(hr_qp); +err_out: + mutex_destroy(&hr_qp->mutex); return ret; } @@ -1234,6 +1236,7 @@ void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, free_qp_buf(hr_dev, hr_qp); free_kernel_wrid(hr_qp); free_qp_db(hr_dev, hr_qp, udata); + mutex_destroy(&hr_qp->mutex); } static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, @@ -1271,7 +1274,6 @@ int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, struct ib_device *ibdev = qp->device; struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); struct hns_roce_qp *hr_qp = to_hr_qp(qp); - struct ib_pd *pd = qp->pd; int ret; ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata); @@ -1286,7 +1288,7 @@ int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; } - ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); + ret = hns_roce_create_qp_common(hr_dev, init_attr, udata, hr_qp); if (ret) ibdev_err(ibdev, "create QP type 0x%x failed(%d)\n", init_attr->qp_type, ret); @@ -1386,6 +1388,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_ib_modify_qp_resp resp = {}; struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); enum ib_qp_state cur_state, new_state; int ret = -EINVAL; @@ -1427,6 +1430,18 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, new_state, udata); + if (ret) + goto out; + + if (udata && udata->outlen) { + resp.tc_mode = hr_qp->tc_mode; + resp.priority = hr_qp->sl; + ret = ib_copy_to_udata(udata, &resp, + min(udata->outlen, sizeof(resp))); + if (ret) + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to copy modify qp resp.\n"); + } out: mutex_unlock(&hr_qp->mutex); @@ -1561,5 +1576,7 @@ void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) ida_destroy(&hr_dev->qp_table.bank[i].ida); + mutex_destroy(&hr_dev->qp_table.bank_mutex); + mutex_destroy(&hr_dev->qp_table.scc_mutex); kfree(hr_dev->qp_table.idx_table.spare_idx); } diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 4abae9477854..f1997abc97ca 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -123,7 +123,7 @@ static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) return ret; } - ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL)); + ret = xa_err(xa_store_irq(&srq_table->xa, srq->srqn, srq, GFP_KERNEL)); if (ret) { ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret); goto err_put; @@ -136,7 +136,7 @@ static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) return 0; err_xa: - xa_erase(&srq_table->xa, srq->srqn); + xa_erase_irq(&srq_table->xa, srq->srqn); err_put: hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); @@ -154,7 +154,7 @@ static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) dev_err(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n", ret, srq->srqn); - xa_erase(&srq_table->xa, srq->srqn); + xa_erase_irq(&srq_table->xa, srq->srqn); if (refcount_dec_and_test(&srq->refcount)) complete(&srq->free); @@ -250,7 +250,7 @@ static void free_srq_wqe_buf(struct hns_roce_dev *hr_dev, hns_roce_mtr_destroy(hr_dev, &srq->buf_mtr); } -static int alloc_srq_wrid(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) +static int alloc_srq_wrid(struct hns_roce_srq *srq) { srq->wrid = kvmalloc_array(srq->wqe_cnt, sizeof(u64), GFP_KERNEL); if (!srq->wrid) @@ -366,7 +366,7 @@ static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, goto err_idx; if (!udata) { - ret = alloc_srq_wrid(hr_dev, srq); + ret = alloc_srq_wrid(srq); if (ret) goto err_wqe_buf; } @@ -518,6 +518,7 @@ err_srq_db: err_srq_buf: free_srq_buf(hr_dev, srq); err_out: + mutex_destroy(&srq->mutex); atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_SRQ_CREATE_ERR_CNT]); return ret; @@ -532,6 +533,7 @@ int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) free_srqn(hr_dev, srq); free_srq_db(hr_dev, srq, udata); free_srq_buf(hr_dev, srq); + mutex_destroy(&srq->mutex); return 0; } diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index 4a71e678d09c..c6a3fd57a196 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -9,23 +9,23 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); + struct mana_ib_create_cq_resp resp = {}; + struct mana_ib_ucontext *mana_ucontext; struct ib_device *ibdev = ibcq->device; struct mana_ib_create_cq ucmd = {}; struct mana_ib_dev *mdev; - struct gdma_context *gc; + bool is_rnic_cq; + u32 doorbell; int err; mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); - gc = mdev_to_gc(mdev); - if (udata->inlen < sizeof(ucmd)) - return -EINVAL; + cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors; + cq->cq_handle = INVALID_MANA_HANDLE; - if (attr->comp_vector > gc->max_num_queues) + if (udata->inlen < offsetof(struct mana_ib_create_cq, flags)) return -EINVAL; - cq->comp_vector = attr->comp_vector; - err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); if (err) { ibdev_dbg(ibdev, @@ -33,42 +33,54 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, return err; } - if (attr->cqe > mdev->adapter_caps.max_qp_wr) { + is_rnic_cq = !!(ucmd.flags & MANA_IB_CREATE_RNIC_CQ); + + if (!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) { ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); return -EINVAL; } cq->cqe = attr->cqe; - cq->umem = ib_umem_get(ibdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(cq->umem)) { - err = PTR_ERR(cq->umem); - ibdev_dbg(ibdev, "Failed to get umem for create cq, err %d\n", - err); + err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, &cq->queue); + if (err) { + ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err); return err; } - err = mana_ib_create_zero_offset_dma_region(mdev, cq->umem, &cq->gdma_region); - if (err) { - ibdev_dbg(ibdev, - "Failed to create dma region for create cq, %d\n", - err); - goto err_release_umem; + mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, + ibucontext); + doorbell = mana_ucontext->doorbell; + + if (is_rnic_cq) { + err = mana_ib_gd_create_cq(mdev, cq, doorbell); + if (err) { + ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err); + goto err_destroy_queue; + } + + err = mana_ib_install_cq_cb(mdev, cq); + if (err) { + ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err); + goto err_destroy_rnic_cq; + } } - ibdev_dbg(ibdev, - "create_dma_region ret %d gdma_region 0x%llx\n", - err, cq->gdma_region); - - /* - * The CQ ID is not known at this time. The ID is generated at create_qp - */ - cq->id = INVALID_QUEUE_ID; + resp.cqid = cq->queue.id; + err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); + if (err) { + ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); + goto err_remove_cq_cb; + } return 0; -err_release_umem: - ib_umem_release(cq->umem); +err_remove_cq_cb: + mana_ib_remove_cq_cb(mdev, cq); +err_destroy_rnic_cq: + mana_ib_gd_destroy_cq(mdev, cq); +err_destroy_queue: + mana_ib_destroy_queue(mdev, &cq->queue); + return err; } @@ -77,25 +89,17 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); struct ib_device *ibdev = ibcq->device; struct mana_ib_dev *mdev; - struct gdma_context *gc; - int err; mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); - gc = mdev_to_gc(mdev); - err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region); - if (err) { - ibdev_dbg(ibdev, - "Failed to destroy dma region, %d\n", err); - return err; - } + mana_ib_remove_cq_cb(mdev, cq); - if (cq->id != INVALID_QUEUE_ID) { - kfree(gc->cq_table[cq->id]); - gc->cq_table[cq->id] = NULL; - } + /* Ignore return code as there is not much we can do about it. + * The error message is printed inside. + */ + mana_ib_gd_destroy_cq(mdev, cq); - ib_umem_release(cq->umem); + mana_ib_destroy_queue(mdev, &cq->queue); return 0; } @@ -113,8 +117,10 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) struct gdma_context *gc = mdev_to_gc(mdev); struct gdma_queue *gdma_cq; + if (cq->queue.id >= gc->max_num_cqs) + return -EINVAL; /* Create CQ table entry */ - WARN_ON(gc->cq_table[cq->id]); + WARN_ON(gc->cq_table[cq->queue.id]); gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL); if (!gdma_cq) return -ENOMEM; @@ -122,7 +128,18 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) gdma_cq->cq.context = cq; gdma_cq->type = GDMA_CQ; gdma_cq->cq.callback = mana_ib_cq_handler; - gdma_cq->id = cq->id; - gc->cq_table[cq->id] = gdma_cq; + gdma_cq->id = cq->queue.id; + gc->cq_table[cq->queue.id] = gdma_cq; return 0; } + +void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) +{ + struct gdma_context *gc = mdev_to_gc(mdev); + + if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID) + return; + + kfree(gc->cq_table[cq->queue.id]); + gc->cq_table[cq->queue.id] = NULL; +} diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index 6fa902ee80a6..7e09ceb3da53 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -15,6 +15,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .driver_id = RDMA_DRIVER_MANA, .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, + .add_gid = mana_ib_gd_add_gid, .alloc_pd = mana_ib_alloc_pd, .alloc_ucontext = mana_ib_alloc_ucontext, .create_cq = mana_ib_create_cq, @@ -23,18 +24,21 @@ static const struct ib_device_ops mana_ib_dev_ops = { .create_wq = mana_ib_create_wq, .dealloc_pd = mana_ib_dealloc_pd, .dealloc_ucontext = mana_ib_dealloc_ucontext, + .del_gid = mana_ib_gd_del_gid, .dereg_mr = mana_ib_dereg_mr, .destroy_cq = mana_ib_destroy_cq, .destroy_qp = mana_ib_destroy_qp, .destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table, .destroy_wq = mana_ib_destroy_wq, .disassociate_ucontext = mana_ib_disassociate_ucontext, + .get_link_layer = mana_ib_get_link_layer, .get_port_immutable = mana_ib_get_port_immutable, .mmap = mana_ib_mmap, .modify_qp = mana_ib_modify_qp, .modify_wq = mana_ib_modify_wq, .query_device = mana_ib_query_device, .query_gid = mana_ib_query_gid, + .query_pkey = mana_ib_query_pkey, .query_port = mana_ib_query_port, .reg_user_mr = mana_ib_reg_user_mr, @@ -51,8 +55,10 @@ static int mana_ib_probe(struct auxiliary_device *adev, { struct mana_adev *madev = container_of(adev, struct mana_adev, adev); struct gdma_dev *mdev = madev->mdev; + struct net_device *upper_ndev; struct mana_context *mc; struct mana_ib_dev *dev; + u8 mac_addr[ETH_ALEN]; int ret; mc = mdev->driver_data; @@ -74,9 +80,25 @@ static int mana_ib_probe(struct auxiliary_device *adev, * num_comp_vectors needs to set to the max MSIX index * when interrupts and event queues are implemented */ - dev->ib_dev.num_comp_vectors = 1; + dev->ib_dev.num_comp_vectors = mdev->gdma_context->max_num_queues; dev->ib_dev.dev.parent = mdev->gdma_context->dev; + rcu_read_lock(); /* required to get upper dev */ + upper_ndev = netdev_master_upper_dev_get_rcu(mc->ports[0]); + if (!upper_ndev) { + rcu_read_unlock(); + ret = -ENODEV; + ibdev_err(&dev->ib_dev, "Failed to get master netdev"); + goto free_ib_device; + } + ether_addr_copy(mac_addr, upper_ndev->dev_addr); + ret = ib_device_set_netdev(&dev->ib_dev, upper_ndev, 1); + rcu_read_unlock(); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret); + goto free_ib_device; + } + ret = mana_gd_register_device(&mdev->gdma_context->mana_ib); if (ret) { ibdev_err(&dev->ib_dev, "Failed to register device, ret %d", @@ -92,15 +114,36 @@ static int mana_ib_probe(struct auxiliary_device *adev, goto deregister_device; } + ret = mana_ib_create_eqs(dev); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret); + goto deregister_device; + } + + ret = mana_ib_gd_create_rnic_adapter(dev); + if (ret) + goto destroy_eqs; + + ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", + ret); + goto destroy_rnic; + } + ret = ib_register_device(&dev->ib_dev, "mana_%d", mdev->gdma_context->dev); if (ret) - goto deregister_device; + goto destroy_rnic; dev_set_drvdata(&adev->dev, dev); return 0; +destroy_rnic: + mana_ib_gd_destroy_rnic_adapter(dev); +destroy_eqs: + mana_ib_destroy_eqs(dev); deregister_device: mana_gd_deregister_device(dev->gdma_dev); free_ib_device: @@ -113,9 +156,9 @@ static void mana_ib_remove(struct auxiliary_device *adev) struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev); ib_unregister_device(&dev->ib_dev); - + mana_ib_gd_destroy_rnic_adapter(dev); + mana_ib_destroy_eqs(dev); mana_gd_deregister_device(dev->gdma_dev); - ib_dealloc_device(&dev->ib_dev); } diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 71e33feee61b..2a411357640e 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -237,6 +237,47 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret); } +int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, + struct mana_ib_queue *queue) +{ + struct ib_umem *umem; + int err; + + queue->umem = NULL; + queue->id = INVALID_QUEUE_ID; + queue->gdma_region = GDMA_INVALID_DMA_REGION; + + umem = ib_umem_get(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(umem)) { + err = PTR_ERR(umem); + ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %d\n", err); + return err; + } + + err = mana_ib_create_zero_offset_dma_region(mdev, umem, &queue->gdma_region); + if (err) { + ibdev_dbg(&mdev->ib_dev, "Failed to create dma region, %d\n", err); + goto free_umem; + } + queue->umem = umem; + + ibdev_dbg(&mdev->ib_dev, "created dma region 0x%llx\n", queue->gdma_region); + + return 0; +free_umem: + ib_umem_release(umem); + return err; +} + +void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue) +{ + /* Ignore return code as there is not much we can do about it. + * The error message is printed inside. + */ + mana_ib_gd_destroy_dma_region(mdev, queue->gdma_region); + ib_umem_release(queue->umem); +} + static int mana_ib_gd_first_dma_region(struct mana_ib_dev *dev, struct gdma_context *gc, @@ -484,11 +525,18 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { - /* - * This version only support RAW_PACKET - * other values need to be filled for other types - */ + struct ib_port_attr attr; + int err; + + err = ib_query_port(ibdev, port_num, &attr); + if (err) + return err; + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; + if (port_num == 1) + immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; return 0; } @@ -514,7 +562,42 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, int mana_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { - /* This version doesn't return port properties */ + struct net_device *ndev = mana_ib_get_netdev(ibdev, port); + + if (!ndev) + return -EINVAL; + + memset(props, 0, sizeof(*props)); + props->max_mtu = IB_MTU_4096; + props->active_mtu = ib_mtu_int_to_enum(ndev->mtu); + + if (netif_carrier_ok(ndev) && netif_running(ndev)) { + props->state = IB_PORT_ACTIVE; + props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; + } else { + props->state = IB_PORT_DOWN; + props->phys_state = IB_PORT_PHYS_STATE_DISABLED; + } + + props->active_width = IB_WIDTH_4X; + props->active_speed = IB_SPEED_EDR; + props->pkey_tbl_len = 1; + if (port == 1) + props->gid_tbl_len = 16; + + return 0; +} + +enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num) +{ + return IB_LINK_LAYER_ETHERNET; +} + +int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) +{ + if (index != 0) + return -EINVAL; + *pkey = IB_DEFAULT_PKEY_FULL; return 0; } @@ -570,3 +653,238 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev) return 0; } + +int mana_ib_create_eqs(struct mana_ib_dev *mdev) +{ + struct gdma_context *gc = mdev_to_gc(mdev); + struct gdma_queue_spec spec = {}; + int err, i; + + spec.type = GDMA_EQ; + spec.monitor_avl_buf = false; + spec.queue_size = EQ_SIZE; + spec.eq.callback = NULL; + spec.eq.context = mdev; + spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; + spec.eq.msix_index = 0; + + err = mana_gd_create_mana_eq(&gc->mana_ib, &spec, &mdev->fatal_err_eq); + if (err) + return err; + + mdev->eqs = kcalloc(mdev->ib_dev.num_comp_vectors, sizeof(struct gdma_queue *), + GFP_KERNEL); + if (!mdev->eqs) { + err = -ENOMEM; + goto destroy_fatal_eq; + } + + for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++) { + spec.eq.msix_index = (i + 1) % gc->num_msix_usable; + err = mana_gd_create_mana_eq(mdev->gdma_dev, &spec, &mdev->eqs[i]); + if (err) + goto destroy_eqs; + } + + return 0; + +destroy_eqs: + while (i-- > 0) + mana_gd_destroy_queue(gc, mdev->eqs[i]); + kfree(mdev->eqs); +destroy_fatal_eq: + mana_gd_destroy_queue(gc, mdev->fatal_err_eq); + return err; +} + +void mana_ib_destroy_eqs(struct mana_ib_dev *mdev) +{ + struct gdma_context *gc = mdev_to_gc(mdev); + int i; + + mana_gd_destroy_queue(gc, mdev->fatal_err_eq); + + for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++) + mana_gd_destroy_queue(gc, mdev->eqs[i]); + + kfree(mdev->eqs); +} + +int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev) +{ + struct mana_rnic_create_adapter_resp resp = {}; + struct mana_rnic_create_adapter_req req = {}; + struct gdma_context *gc = mdev_to_gc(mdev); + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER, sizeof(req), sizeof(resp)); + req.hdr.req.msg_version = GDMA_MESSAGE_V2; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create RNIC adapter err %d", err); + return err; + } + mdev->adapter_handle = resp.adapter; + + return 0; +} + +int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev) +{ + struct mana_rnic_destroy_adapter_resp resp = {}; + struct mana_rnic_destroy_adapter_req req = {}; + struct gdma_context *gc; + int err; + + gc = mdev_to_gc(mdev); + mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_ADAPTER, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to destroy RNIC adapter err %d", err); + return err; + } + + return 0; +} + +int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context) +{ + struct mana_ib_dev *mdev = container_of(attr->device, struct mana_ib_dev, ib_dev); + enum rdma_network_type ntype = rdma_gid_attr_network_type(attr); + struct mana_rnic_config_addr_resp resp = {}; + struct gdma_context *gc = mdev_to_gc(mdev); + struct mana_rnic_config_addr_req req = {}; + int err; + + if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6) { + ibdev_dbg(&mdev->ib_dev, "Unsupported rdma network type %d", ntype); + return -EINVAL; + } + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.op = ADDR_OP_ADD; + req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4; + copy_in_reverse(req.ip_addr, attr->gid.raw, sizeof(union ib_gid)); + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to config IP addr err %d\n", err); + return err; + } + + return 0; +} + +int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context) +{ + struct mana_ib_dev *mdev = container_of(attr->device, struct mana_ib_dev, ib_dev); + enum rdma_network_type ntype = rdma_gid_attr_network_type(attr); + struct mana_rnic_config_addr_resp resp = {}; + struct gdma_context *gc = mdev_to_gc(mdev); + struct mana_rnic_config_addr_req req = {}; + int err; + + if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6) { + ibdev_dbg(&mdev->ib_dev, "Unsupported rdma network type %d", ntype); + return -EINVAL; + } + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.op = ADDR_OP_REMOVE; + req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4; + copy_in_reverse(req.ip_addr, attr->gid.raw, sizeof(union ib_gid)); + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to config IP addr err %d\n", err); + return err; + } + + return 0; +} + +int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac) +{ + struct mana_rnic_config_mac_addr_resp resp = {}; + struct mana_rnic_config_mac_addr_req req = {}; + struct gdma_context *gc = mdev_to_gc(mdev); + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_MAC_ADDR, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.op = op; + copy_in_reverse(req.mac_addr, mac, ETH_ALEN); + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to config Mac addr err %d", err); + return err; + } + + return 0; +} + +int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell) +{ + struct gdma_context *gc = mdev_to_gc(mdev); + struct mana_rnic_create_cq_resp resp = {}; + struct mana_rnic_create_cq_req req = {}; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_CQ, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.gdma_region = cq->queue.gdma_region; + req.eq_id = mdev->eqs[cq->comp_vector]->id; + req.doorbell_page = doorbell; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create cq err %d", err); + return err; + } + + cq->queue.id = resp.cq_id; + cq->cq_handle = resp.cq_handle; + /* The GDMA region is now owned by the CQ handle */ + cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; + + return 0; +} + +int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) +{ + struct gdma_context *gc = mdev_to_gc(mdev); + struct mana_rnic_destroy_cq_resp resp = {}; + struct mana_rnic_destroy_cq_req req = {}; + int err; + + if (cq->cq_handle == INVALID_MANA_HANDLE) + return 0; + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_CQ, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.cq_handle = cq->cq_handle; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to destroy cq err %d", err); + return err; + } + + return 0; +} diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index f83390eebb7d..68c3b4f0faa4 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -45,19 +45,26 @@ struct mana_ib_adapter_caps { u32 max_inline_data_size; }; +struct mana_ib_queue { + struct ib_umem *umem; + u64 gdma_region; + u64 id; +}; + struct mana_ib_dev { struct ib_device ib_dev; struct gdma_dev *gdma_dev; + mana_handle_t adapter_handle; + struct gdma_queue *fatal_err_eq; + struct gdma_queue **eqs; struct mana_ib_adapter_caps adapter_caps; }; struct mana_ib_wq { struct ib_wq ibwq; - struct ib_umem *umem; + struct mana_ib_queue queue; int wqe; u32 wq_buf_size; - u64 gdma_region; - u64 id; mana_handle_t rx_object; }; @@ -82,22 +89,17 @@ struct mana_ib_mr { struct mana_ib_cq { struct ib_cq ibcq; - struct ib_umem *umem; + struct mana_ib_queue queue; int cqe; - u64 gdma_region; - u64 id; u32 comp_vector; + mana_handle_t cq_handle; }; struct mana_ib_qp { struct ib_qp ibqp; - /* Work queue info */ - struct ib_umem *sq_umem; - int sqe; - u64 sq_gdma_region; - u64 sq_id; - mana_handle_t tx_object; + mana_handle_t qp_handle; + struct mana_ib_queue raw_sq; /* The port on the IB device, starting with 1 */ u32 port; @@ -114,6 +116,12 @@ struct mana_ib_rwq_ind_table { enum mana_ib_command_code { MANA_IB_GET_ADAPTER_CAP = 0x30001, + MANA_IB_CREATE_ADAPTER = 0x30002, + MANA_IB_DESTROY_ADAPTER = 0x30003, + MANA_IB_CONFIG_IP_ADDR = 0x30004, + MANA_IB_CONFIG_MAC_ADDR = 0x30005, + MANA_IB_CREATE_CQ = 0x30008, + MANA_IB_DESTROY_CQ = 0x30009, }; struct mana_ib_query_adapter_caps_req { @@ -142,6 +150,86 @@ struct mana_ib_query_adapter_caps_resp { u32 max_inline_data_size; }; /* HW Data */ +struct mana_rnic_create_adapter_req { + struct gdma_req_hdr hdr; + u32 notify_eq_id; + u32 reserved; + u64 feature_flags; +}; /*HW Data */ + +struct mana_rnic_create_adapter_resp { + struct gdma_resp_hdr hdr; + mana_handle_t adapter; +}; /* HW Data */ + +struct mana_rnic_destroy_adapter_req { + struct gdma_req_hdr hdr; + mana_handle_t adapter; +}; /*HW Data */ + +struct mana_rnic_destroy_adapter_resp { + struct gdma_resp_hdr hdr; +}; /* HW Data */ + +enum mana_ib_addr_op { + ADDR_OP_ADD = 1, + ADDR_OP_REMOVE = 2, +}; + +enum sgid_entry_type { + SGID_TYPE_IPV4 = 1, + SGID_TYPE_IPV6 = 2, +}; + +struct mana_rnic_config_addr_req { + struct gdma_req_hdr hdr; + mana_handle_t adapter; + enum mana_ib_addr_op op; + enum sgid_entry_type sgid_type; + u8 ip_addr[16]; +}; /* HW Data */ + +struct mana_rnic_config_addr_resp { + struct gdma_resp_hdr hdr; +}; /* HW Data */ + +struct mana_rnic_config_mac_addr_req { + struct gdma_req_hdr hdr; + mana_handle_t adapter; + enum mana_ib_addr_op op; + u8 mac_addr[ETH_ALEN]; + u8 reserved[6]; +}; /* HW Data */ + +struct mana_rnic_config_mac_addr_resp { + struct gdma_resp_hdr hdr; +}; /* HW Data */ + +struct mana_rnic_create_cq_req { + struct gdma_req_hdr hdr; + mana_handle_t adapter; + u64 gdma_region; + u32 eq_id; + u32 doorbell_page; +}; /* HW Data */ + +struct mana_rnic_create_cq_resp { + struct gdma_resp_hdr hdr; + mana_handle_t cq_handle; + u32 cq_id; + u32 reserved; +}; /* HW Data */ + +struct mana_rnic_destroy_cq_req { + struct gdma_req_hdr hdr; + mana_handle_t adapter; + mana_handle_t cq_handle; +}; /* HW Data */ + +struct mana_rnic_destroy_cq_resp { + struct gdma_resp_hdr hdr; +}; /* HW Data */ + static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev) { return mdev->gdma_dev->gdma_context; @@ -158,7 +246,16 @@ static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 return mc->ports[port - 1]; } +static inline void copy_in_reverse(u8 *dst, const u8 *src, u32 size) +{ + u32 i; + + for (i = 0; i < size; i++) + dst[size - 1 - i] = src[i]; +} + int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); +void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, mana_handle_t *gdma_region); @@ -169,6 +266,10 @@ int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, mana_handle_t gdma_region); +int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, + struct mana_ib_queue *queue); +void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue); + struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, struct ib_wq_init_attr *init_attr, struct ib_udata *udata); @@ -231,4 +332,26 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index, void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext); int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev); + +int mana_ib_create_eqs(struct mana_ib_dev *mdev); + +void mana_ib_destroy_eqs(struct mana_ib_dev *mdev); + +int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev); + +int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev); + +int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); + +enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num); + +int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context); + +int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context); + +int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac); + +int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell); + +int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); #endif diff --git a/drivers/infiniband/hw/mana/mr.c b/drivers/infiniband/hw/mana/mr.c index b70b13484f09..4f13423ecdbd 100644 --- a/drivers/infiniband/hw/mana/mr.c +++ b/drivers/infiniband/hw/mana/mr.c @@ -135,7 +135,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, } ibdev_dbg(ibdev, - "create_dma_region ret %d gdma_region %llx\n", err, + "created dma region for user-mr 0x%llx\n", dma_region_handle); mr_params.pd_handle = pd->pd_handle; diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 258f89464c10..ba13c5abf8ef 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -95,11 +95,9 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); struct mana_ib_dev *mdev = container_of(pd->device, struct mana_ib_dev, ib_dev); - struct gdma_context *gc = mdev_to_gc(mdev); struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl; struct mana_ib_create_qp_rss_resp resp = {}; struct mana_ib_create_qp_rss ucmd = {}; - struct gdma_queue **gdma_cq_allocated; mana_handle_t *mana_ind_table; struct mana_port_context *mpc; unsigned int ind_tbl_size; @@ -173,13 +171,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, goto fail; } - gdma_cq_allocated = kcalloc(ind_tbl_size, sizeof(*gdma_cq_allocated), - GFP_KERNEL); - if (!gdma_cq_allocated) { - ret = -ENOMEM; - goto fail; - } - qp->port = port; for (i = 0; i < ind_tbl_size; i++) { @@ -192,13 +183,13 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, ibcq = ibwq->cq; cq = container_of(ibcq, struct mana_ib_cq, ibcq); - wq_spec.gdma_region = wq->gdma_region; + wq_spec.gdma_region = wq->queue.gdma_region; wq_spec.queue_size = wq->wq_buf_size; - cq_spec.gdma_region = cq->gdma_region; + cq_spec.gdma_region = cq->queue.gdma_region; cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE; cq_spec.modr_ctx_id = 0; - eq = &mpc->ac->eqs[cq->comp_vector % gc->max_num_queues]; + eq = &mpc->ac->eqs[cq->comp_vector]; cq_spec.attached_eq = eq->eq->id; ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ, @@ -210,18 +201,18 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, } /* The GDMA regions are now owned by the WQ object */ - wq->gdma_region = GDMA_INVALID_DMA_REGION; - cq->gdma_region = GDMA_INVALID_DMA_REGION; + wq->queue.gdma_region = GDMA_INVALID_DMA_REGION; + cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; - wq->id = wq_spec.queue_index; - cq->id = cq_spec.queue_index; + wq->queue.id = wq_spec.queue_index; + cq->queue.id = cq_spec.queue_index; ibdev_dbg(&mdev->ib_dev, - "ret %d rx_object 0x%llx wq id %llu cq id %llu\n", - ret, wq->rx_object, wq->id, cq->id); + "rx_object 0x%llx wq id %llu cq id %llu\n", + wq->rx_object, wq->queue.id, cq->queue.id); - resp.entries[i].cqid = cq->id; - resp.entries[i].wqid = wq->id; + resp.entries[i].cqid = cq->queue.id; + resp.entries[i].wqid = wq->queue.id; mana_ind_table[i] = wq->rx_object; @@ -229,8 +220,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, ret = mana_ib_install_cq_cb(mdev, cq); if (ret) goto fail; - - gdma_cq_allocated[i] = gc->cq_table[cq->id]; } resp.num_entries = i; @@ -250,7 +239,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, goto fail; } - kfree(gdma_cq_allocated); kfree(mana_ind_table); return 0; @@ -262,13 +250,10 @@ fail: wq = container_of(ibwq, struct mana_ib_wq, ibwq); cq = container_of(ibcq, struct mana_ib_cq, ibcq); - gc->cq_table[cq->id] = NULL; - kfree(gdma_cq_allocated[i]); - + mana_ib_remove_cq_cb(mdev, cq); mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); } - kfree(gdma_cq_allocated); kfree(mana_ind_table); return ret; @@ -287,15 +272,12 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, struct mana_ib_ucontext *mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext); - struct gdma_context *gc = mdev_to_gc(mdev); struct mana_ib_create_qp_resp resp = {}; struct mana_ib_create_qp ucmd = {}; - struct gdma_queue *gdma_cq = NULL; struct mana_obj_spec wq_spec = {}; struct mana_obj_spec cq_spec = {}; struct mana_port_context *mpc; struct net_device *ndev; - struct ib_umem *umem; struct mana_eq *eq; int eq_vec; u32 port; @@ -344,56 +326,39 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n", ucmd.sq_buf_addr, ucmd.port); - umem = ib_umem_get(ibpd->device, ucmd.sq_buf_addr, ucmd.sq_buf_size, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) { - err = PTR_ERR(umem); - ibdev_dbg(&mdev->ib_dev, - "Failed to get umem for create qp-raw, err %d\n", - err); - goto err_free_vport; - } - qp->sq_umem = umem; - - err = mana_ib_create_zero_offset_dma_region(mdev, qp->sq_umem, - &qp->sq_gdma_region); + err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq); if (err) { ibdev_dbg(&mdev->ib_dev, - "Failed to create dma region for create qp-raw, %d\n", - err); - goto err_release_umem; + "Failed to create queue for create qp-raw, err %d\n", err); + goto err_free_vport; } - ibdev_dbg(&mdev->ib_dev, - "create_dma_region ret %d gdma_region 0x%llx\n", - err, qp->sq_gdma_region); - /* Create a WQ on the same port handle used by the Ethernet */ - wq_spec.gdma_region = qp->sq_gdma_region; + wq_spec.gdma_region = qp->raw_sq.gdma_region; wq_spec.queue_size = ucmd.sq_buf_size; - cq_spec.gdma_region = send_cq->gdma_region; + cq_spec.gdma_region = send_cq->queue.gdma_region; cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE; cq_spec.modr_ctx_id = 0; - eq_vec = send_cq->comp_vector % gc->max_num_queues; + eq_vec = send_cq->comp_vector; eq = &mpc->ac->eqs[eq_vec]; cq_spec.attached_eq = eq->eq->id; err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec, - &cq_spec, &qp->tx_object); + &cq_spec, &qp->qp_handle); if (err) { ibdev_dbg(&mdev->ib_dev, "Failed to create wq for create raw-qp, err %d\n", err); - goto err_destroy_dma_region; + goto err_destroy_queue; } /* The GDMA regions are now owned by the WQ object */ - qp->sq_gdma_region = GDMA_INVALID_DMA_REGION; - send_cq->gdma_region = GDMA_INVALID_DMA_REGION; + qp->raw_sq.gdma_region = GDMA_INVALID_DMA_REGION; + send_cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; - qp->sq_id = wq_spec.queue_index; - send_cq->id = cq_spec.queue_index; + qp->raw_sq.id = wq_spec.queue_index; + send_cq->queue.id = cq_spec.queue_index; /* Create CQ table entry */ err = mana_ib_install_cq_cb(mdev, send_cq); @@ -401,11 +366,11 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, goto err_destroy_wq_obj; ibdev_dbg(&mdev->ib_dev, - "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err, - qp->tx_object, qp->sq_id, send_cq->id); + "qp->qp_handle 0x%llx sq id %llu cq id %llu\n", + qp->qp_handle, qp->raw_sq.id, send_cq->queue.id); - resp.sqid = qp->sq_id; - resp.cqid = send_cq->id; + resp.sqid = qp->raw_sq.id; + resp.cqid = send_cq->queue.id; resp.tx_vp_offset = pd->tx_vp_offset; err = ib_copy_to_udata(udata, &resp, sizeof(resp)); @@ -413,23 +378,19 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, ibdev_dbg(&mdev->ib_dev, "Failed copy udata for create qp-raw, %d\n", err); - goto err_release_gdma_cq; + goto err_remove_cq_cb; } return 0; -err_release_gdma_cq: - kfree(gdma_cq); - gc->cq_table[send_cq->id] = NULL; +err_remove_cq_cb: + mana_ib_remove_cq_cb(mdev, send_cq); err_destroy_wq_obj: - mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object); + mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); -err_destroy_dma_region: - mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region); - -err_release_umem: - ib_umem_release(umem); +err_destroy_queue: + mana_ib_destroy_queue(mdev, &qp->raw_sq); err_free_vport: mana_ib_uncfg_vport(mdev, pd, port); @@ -503,12 +464,9 @@ static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata) mpc = netdev_priv(ndev); pd = container_of(ibpd, struct mana_ib_pd, ibpd); - mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object); + mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); - if (qp->sq_umem) { - mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region); - ib_umem_release(qp->sq_umem); - } + mana_ib_destroy_queue(mdev, &qp->raw_sq); mana_ib_uncfg_vport(mdev, pd, qp->port); diff --git a/drivers/infiniband/hw/mana/wq.c b/drivers/infiniband/hw/mana/wq.c index 7c9c69962573..f959f4b9244f 100644 --- a/drivers/infiniband/hw/mana/wq.c +++ b/drivers/infiniband/hw/mana/wq.c @@ -13,7 +13,6 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, container_of(pd->device, struct mana_ib_dev, ib_dev); struct mana_ib_create_wq ucmd = {}; struct mana_ib_wq *wq; - struct ib_umem *umem; int err; if (udata->inlen < sizeof(ucmd)) @@ -32,39 +31,18 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr); - umem = ib_umem_get(pd->device, ucmd.wq_buf_addr, ucmd.wq_buf_size, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) { - err = PTR_ERR(umem); + err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue); + if (err) { ibdev_dbg(&mdev->ib_dev, - "Failed to get umem for create wq, err %d\n", err); + "Failed to create queue for create wq, %d\n", err); goto err_free_wq; } - wq->umem = umem; wq->wqe = init_attr->max_wr; wq->wq_buf_size = ucmd.wq_buf_size; wq->rx_object = INVALID_MANA_HANDLE; - - err = mana_ib_create_zero_offset_dma_region(mdev, wq->umem, &wq->gdma_region); - if (err) { - ibdev_dbg(&mdev->ib_dev, - "Failed to create dma region for create wq, %d\n", - err); - goto err_release_umem; - } - - ibdev_dbg(&mdev->ib_dev, - "create_dma_region ret %d gdma_region 0x%llx\n", - err, wq->gdma_region); - - /* WQ ID is returned at wq_create time, doesn't know the value yet */ - return &wq->ibwq; -err_release_umem: - ib_umem_release(umem); - err_free_wq: kfree(wq); @@ -86,8 +64,7 @@ int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) mdev = container_of(ib_dev, struct mana_ib_dev, ib_dev); - mana_ib_gd_destroy_dma_region(mdev, wq->gdma_region); - ib_umem_release(wq->umem); + mana_ib_destroy_queue(mdev, &wq->queue); kfree(wq); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index c2b557e64290..2366c46eebc8 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -264,8 +264,7 @@ static struct net_device *mlx5_ib_get_netdev(struct ib_device *device, */ read_lock(&ibdev->port[port_num - 1].roce.netdev_lock); ndev = ibdev->port[port_num - 1].roce.netdev; - if (ndev) - dev_hold(ndev); + dev_hold(ndev); read_unlock(&ibdev->port[port_num - 1].roce.netdev_lock); out: diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c index 96ffbbaf0a73..5a22be14d958 100644 --- a/drivers/infiniband/hw/mlx5/mem.c +++ b/drivers/infiniband/hw/mlx5/mem.c @@ -30,6 +30,7 @@ * SOFTWARE. */ +#include <linux/io.h> #include <rdma/ib_umem_odp.h> #include "mlx5_ib.h" #include <linux/jiffies.h> @@ -108,7 +109,6 @@ static int post_send_nop(struct mlx5_ib_dev *dev, struct ib_qp *ibqp, u64 wr_id, __be32 mmio_wqe[16] = {}; unsigned long flags; unsigned int idx; - int i; if (unlikely(dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)) return -EIO; @@ -148,10 +148,8 @@ static int post_send_nop(struct mlx5_ib_dev *dev, struct ib_qp *ibqp, u64 wr_id, * we hit doorbell */ wmb(); - for (i = 0; i < 8; i++) - mlx5_write64(&mmio_wqe[i * 2], - bf->bfreg->map + bf->offset + i * 8); - io_stop_wc(); + __iowrite64_copy(bf->bfreg->map + bf->offset, mmio_wqe, + sizeof(mmio_wqe) / 8); bf->offset ^= bf->buf_size; diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index a8de35c07c9e..f255a12e26a0 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -643,9 +643,10 @@ struct mlx5_ib_mkey { unsigned int ndescs; struct wait_queue_head wait; refcount_t usecount; - /* User Mkey must hold either a rb_key or a cache_ent. */ + /* Cacheable user Mkey must hold either a rb_key or a cache_ent. */ struct mlx5r_cache_rb_key rb_key; struct mlx5_cache_ent *cache_ent; + u8 cacheable : 1; }; #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE) diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index a8ee2ca1f4a1..ecc111ed5d86 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1158,6 +1158,7 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, if (IS_ERR(mr)) return mr; mr->mmkey.rb_key = rb_key; + mr->mmkey.cacheable = true; return mr; } @@ -1168,6 +1169,7 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, mr->ibmr.pd = pd; mr->umem = umem; mr->page_shift = order_base_2(page_size); + mr->mmkey.cacheable = true; set_mr_fields(dev, mr, umem->length, access_flags, iova); return mr; @@ -1570,7 +1572,8 @@ static bool can_use_umr_rereg_access(struct mlx5_ib_dev *dev, unsigned int diffs = current_access_flags ^ target_access_flags; if (diffs & ~(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | - IB_ACCESS_REMOTE_READ | IB_ACCESS_RELAXED_ORDERING)) + IB_ACCESS_REMOTE_READ | IB_ACCESS_RELAXED_ORDERING | + IB_ACCESS_REMOTE_ATOMIC)) return false; return mlx5r_umr_can_reconfig(dev, current_access_flags, target_access_flags); @@ -1835,6 +1838,23 @@ end: return ret; } +static int mlx5_revoke_mr(struct mlx5_ib_mr *mr) +{ + struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); + struct mlx5_cache_ent *ent = mr->mmkey.cache_ent; + + if (mr->mmkey.cacheable && !mlx5r_umr_revoke_mr(mr) && !cache_ent_find_and_store(dev, mr)) + return 0; + + if (ent) { + spin_lock_irq(&ent->mkeys_queue.lock); + ent->in_use--; + mr->mmkey.cache_ent = NULL; + spin_unlock_irq(&ent->mkeys_queue.lock); + } + return destroy_mkey(dev, mr); +} + int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) { struct mlx5_ib_mr *mr = to_mmr(ibmr); @@ -1880,16 +1900,9 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) } /* Stop DMA */ - if (mr->umem && mlx5r_umr_can_load_pas(dev, mr->umem->length)) - if (mlx5r_umr_revoke_mr(mr) || - cache_ent_find_and_store(dev, mr)) - mr->mmkey.cache_ent = NULL; - - if (!mr->mmkey.cache_ent) { - rc = destroy_mkey(to_mdev(mr->ibmr.device), mr); - if (rc) - return rc; - } + rc = mlx5_revoke_mr(mr); + if (rc) + return rc; if (mr->umem) { bool is_odp = is_odp_mr(mr); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 8115ab107149..e2164f813607 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -3097,7 +3097,6 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, switch (qp->type) { case MLX5_IB_QPT_DCT: err = create_dct(dev, pd, qp, params); - rdma_restrack_no_track(&qp->ibqp.res); break; case MLX5_IB_QPT_DCI: err = create_dci(dev, pd, qp, params); @@ -3109,9 +3108,9 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, err = mlx5_ib_create_gsi(pd, qp, params->attr); break; case MLX5_IB_QPT_HW_GSI: - case MLX5_IB_QPT_REG_UMR: rdma_restrack_no_track(&qp->ibqp.res); fallthrough; + case MLX5_IB_QPT_REG_UMR: default: if (params->udata) err = create_user_qp(dev, pd, qp, params); diff --git a/drivers/infiniband/hw/mlx5/restrack.c b/drivers/infiniband/hw/mlx5/restrack.c index 4ac429e72004..affcf8fe943c 100644 --- a/drivers/infiniband/hw/mlx5/restrack.c +++ b/drivers/infiniband/hw/mlx5/restrack.c @@ -156,6 +156,34 @@ static int fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ibcq) return fill_res_raw(msg, dev, MLX5_SGMT_TYPE_PRM_QUERY_CQ, cq->mcq.cqn); } +static int fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp) +{ + struct mlx5_ib_qp *qp = to_mqp(ibqp); + int ret; + + if (qp->type < IB_QPT_DRIVER) + return 0; + + switch (qp->type) { + case MLX5_IB_QPT_REG_UMR: + ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, + "REG_UMR"); + break; + case MLX5_IB_QPT_DCT: + ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCT"); + break; + case MLX5_IB_QPT_DCI: + ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCI"); + break; + default: + return 0; + } + if (ret) + return ret; + + return nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, IB_QPT_DRIVER); +} + static int fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ibqp) { struct mlx5_ib_dev *dev = to_mdev(ibqp->device); @@ -168,6 +196,7 @@ static const struct ib_device_ops restrack_ops = { .fill_res_cq_entry_raw = fill_res_cq_entry_raw, .fill_res_mr_entry = fill_res_mr_entry, .fill_res_mr_entry_raw = fill_res_mr_entry_raw, + .fill_res_qp_entry = fill_res_qp_entry, .fill_res_qp_entry_raw = fill_res_qp_entry_raw, .fill_stat_mr_entry = fill_stat_mr_entry, }; diff --git a/drivers/infiniband/hw/qib/qib_iba7220.c b/drivers/infiniband/hw/qib/qib_iba7220.c index 6af57067c32e..78dfe98ebcf7 100644 --- a/drivers/infiniband/hw/qib/qib_iba7220.c +++ b/drivers/infiniband/hw/qib/qib_iba7220.c @@ -3281,7 +3281,7 @@ static int qib_7220_intr_fallback(struct qib_devdata *dd) qib_free_irq(dd); dd->msi_lo = 0; - if (pci_alloc_irq_vectors(dd->pcidev, 1, 1, PCI_IRQ_LEGACY) < 0) + if (pci_alloc_irq_vectors(dd->pcidev, 1, 1, PCI_IRQ_INTX) < 0) qib_dev_err(dd, "Failed to enable INTx\n"); qib_setup_7220_interrupt(dd); return 1; diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c index f93906d8fc09..9db29916e35a 100644 --- a/drivers/infiniband/hw/qib/qib_iba7322.c +++ b/drivers/infiniband/hw/qib/qib_iba7322.c @@ -3471,8 +3471,7 @@ try_intx: pci_irq_vector(dd->pcidev, msixnum), ret); qib_7322_free_irq(dd); - pci_alloc_irq_vectors(dd->pcidev, 1, 1, - PCI_IRQ_LEGACY); + pci_alloc_irq_vectors(dd->pcidev, 1, 1, PCI_IRQ_INTX); goto try_intx; } dd->cspec->msix_entries[msixnum].arg = arg; @@ -5143,7 +5142,7 @@ static int qib_7322_intr_fallback(struct qib_devdata *dd) qib_devinfo(dd->pcidev, "MSIx interrupt not detected, trying INTx interrupts\n"); qib_7322_free_irq(dd); - if (pci_alloc_irq_vectors(dd->pcidev, 1, 1, PCI_IRQ_LEGACY) < 0) + if (pci_alloc_irq_vectors(dd->pcidev, 1, 1, PCI_IRQ_INTX) < 0) qib_dev_err(dd, "Failed to enable INTx\n"); qib_setup_7322_interrupt(dd, 0); return 1; diff --git a/drivers/infiniband/hw/qib/qib_pcie.c b/drivers/infiniband/hw/qib/qib_pcie.c index 47bf64ace05c..58c1d62d341b 100644 --- a/drivers/infiniband/hw/qib/qib_pcie.c +++ b/drivers/infiniband/hw/qib/qib_pcie.c @@ -210,7 +210,7 @@ int qib_pcie_params(struct qib_devdata *dd, u32 minw, u32 *nent) } if (dd->flags & QIB_HAS_INTX) - flags |= PCI_IRQ_LEGACY; + flags |= PCI_IRQ_INTX; maxvec = (nent && *nent) ? *nent : 1; nvec = pci_alloc_irq_vectors(dd->pcidev, 1, maxvec, flags); if (nvec < 0) diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c index a5e88185171f..768aad364c89 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c @@ -531,7 +531,7 @@ static int pvrdma_alloc_intrs(struct pvrdma_dev *dev) PCI_IRQ_MSIX); if (ret < 0) { ret = pci_alloc_irq_vectors(pdev, 1, 1, - PCI_IRQ_MSI | PCI_IRQ_LEGACY); + PCI_IRQ_MSI | PCI_IRQ_INTX); if (ret < 0) return ret; } diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index b78b8c0856ab..d48af2180745 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -122,25 +122,16 @@ void retransmit_timer(struct timer_list *t) spin_lock_irqsave(&qp->state_lock, flags); if (qp->valid) { qp->comp.timeout = 1; - rxe_sched_task(&qp->comp.task); + rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); } void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) { - int must_sched; - + rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_SENDER_SCHED); skb_queue_tail(&qp->resp_pkts, skb); - - must_sched = skb_queue_len(&qp->resp_pkts) > 1; - if (must_sched != 0) - rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED); - - if (must_sched) - rxe_sched_task(&qp->comp.task); - else - rxe_run_task(&qp->comp.task); + rxe_sched_task(&qp->send_task); } static inline enum comp_state get_wqe(struct rxe_qp *qp, @@ -325,7 +316,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp, qp->comp.psn = pkt->psn; if (qp->req.wait_psn) { qp->req.wait_psn = 0; - rxe_sched_task(&qp->req.task); + qp->req.again = 1; } } return COMPST_ERROR_RETRY; @@ -476,7 +467,7 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe) */ if (qp->req.wait_fence) { qp->req.wait_fence = 0; - rxe_sched_task(&qp->req.task); + qp->req.again = 1; } } @@ -515,7 +506,7 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, if (qp->req.need_rd_atomic) { qp->comp.timeout_retry = 0; qp->req.need_rd_atomic = 0; - rxe_sched_task(&qp->req.task); + qp->req.again = 1; } } @@ -541,7 +532,7 @@ static inline enum comp_state complete_wqe(struct rxe_qp *qp, if (qp->req.wait_psn) { qp->req.wait_psn = 0; - rxe_sched_task(&qp->req.task); + qp->req.again = 1; } } @@ -654,6 +645,8 @@ int rxe_completer(struct rxe_qp *qp) int ret; unsigned long flags; + qp->req.again = 0; + spin_lock_irqsave(&qp->state_lock, flags); if (!qp->valid || qp_state(qp) == IB_QPS_ERR || qp_state(qp) == IB_QPS_RESET) { @@ -737,7 +730,7 @@ int rxe_completer(struct rxe_qp *qp) if (qp->req.wait_psn) { qp->req.wait_psn = 0; - rxe_sched_task(&qp->req.task); + qp->req.again = 1; } state = COMPST_DONE; @@ -792,7 +785,7 @@ int rxe_completer(struct rxe_qp *qp) RXE_CNT_COMP_RETRY); qp->req.need_retry = 1; qp->comp.started_retry = 1; - rxe_sched_task(&qp->req.task); + qp->req.again = 1; } goto done; @@ -843,8 +836,9 @@ done: ret = 0; goto out; exit: - ret = -EAGAIN; + ret = (qp->req.again) ? 0 : -EAGAIN; out: + qp->req.again = 0; if (pkt) free_pkt(pkt); return ret; diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c index a012522b577a..437917a7d8f2 100644 --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c @@ -14,7 +14,7 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_RCV_RNR].name = "rcvd_rnr_err", [RXE_CNT_SND_RNR].name = "send_rnr_err", [RXE_CNT_RCV_SEQ_ERR].name = "rcvd_seq_err", - [RXE_CNT_COMPLETER_SCHED].name = "ack_deferred", + [RXE_CNT_SENDER_SCHED].name = "ack_deferred", [RXE_CNT_RETRY_EXCEEDED].name = "retry_exceeded_err", [RXE_CNT_RNR_RETRY_EXCEEDED].name = "retry_rnr_exceeded_err", [RXE_CNT_COMP_RETRY].name = "completer_retry_err", diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/infiniband/sw/rxe/rxe_hw_counters.h index 71f4d4fa9dc8..051f9e1c3852 100644 --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h @@ -18,7 +18,7 @@ enum rxe_counters { RXE_CNT_RCV_RNR, RXE_CNT_SND_RNR, RXE_CNT_RCV_SEQ_ERR, - RXE_CNT_COMPLETER_SCHED, + RXE_CNT_SENDER_SCHED, RXE_CNT_RETRY_EXCEEDED, RXE_CNT_RNR_RETRY_EXCEEDED, RXE_CNT_COMP_RETRY, diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index 746110898a0e..ded46119151b 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -164,7 +164,8 @@ void rxe_dealloc(struct ib_device *ib_dev); int rxe_completer(struct rxe_qp *qp); int rxe_requester(struct rxe_qp *qp); -int rxe_responder(struct rxe_qp *qp); +int rxe_sender(struct rxe_qp *qp); +int rxe_receiver(struct rxe_qp *qp); /* rxe_icrc.c */ int rxe_icrc_init(struct rxe_dev *rxe); diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index cd59666158b1..ca9a82e1c4c7 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -345,46 +345,52 @@ int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt, static void rxe_skb_tx_dtor(struct sk_buff *skb) { - struct sock *sk = skb->sk; - struct rxe_qp *qp = sk->sk_user_data; - int skb_out = atomic_dec_return(&qp->skb_out); + struct net_device *ndev = skb->dev; + struct rxe_dev *rxe; + unsigned int qp_index; + struct rxe_qp *qp; + int skb_out; + + rxe = rxe_get_dev_from_net(ndev); + if (!rxe && is_vlan_dev(ndev)) + rxe = rxe_get_dev_from_net(vlan_dev_real_dev(ndev)); + if (WARN_ON(!rxe)) + return; + + qp_index = (int)(uintptr_t)skb->sk->sk_user_data; + if (!qp_index) + return; - if (unlikely(qp->need_req_skb && - skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW)) - rxe_sched_task(&qp->req.task); + qp = rxe_pool_get_index(&rxe->qp_pool, qp_index); + if (!qp) + goto put_dev; + + skb_out = atomic_dec_return(&qp->skb_out); + if (qp->need_req_skb && skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW) + rxe_sched_task(&qp->send_task); rxe_put(qp); +put_dev: + ib_device_put(&rxe->ib_dev); + sock_put(skb->sk); } static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt) { int err; + struct sock *sk = pkt->qp->sk->sk; + sock_hold(sk); + skb->sk = sk; skb->destructor = rxe_skb_tx_dtor; - skb->sk = pkt->qp->sk->sk; - - rxe_get(pkt->qp); atomic_inc(&pkt->qp->skb_out); - if (skb->protocol == htons(ETH_P_IP)) { + if (skb->protocol == htons(ETH_P_IP)) err = ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb); - } else if (skb->protocol == htons(ETH_P_IPV6)) { + else err = ip6_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb); - } else { - rxe_dbg_qp(pkt->qp, "Unknown layer 3 protocol: %d\n", - skb->protocol); - atomic_dec(&pkt->qp->skb_out); - rxe_put(pkt->qp); - kfree_skb(skb); - return -EINVAL; - } - - if (unlikely(net_xmit_eval(err))) { - rxe_dbg_qp(pkt->qp, "error sending packet: %d\n", err); - return -EAGAIN; - } - return 0; + return err; } /* fix up a send packet to match the packets @@ -392,8 +398,15 @@ static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt) */ static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt) { + struct sock *sk = pkt->qp->sk->sk; + memcpy(SKB_TO_PKT(skb), pkt, sizeof(*pkt)); + sock_hold(sk); + skb->sk = sk; + skb->destructor = rxe_skb_tx_dtor; + atomic_inc(&pkt->qp->skb_out); + if (skb->protocol == htons(ETH_P_IP)) skb_pull(skb, sizeof(struct iphdr)); else @@ -440,12 +453,6 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, return err; } - if ((qp_type(qp) != IB_QPT_RC) && - (pkt->mask & RXE_END_MASK)) { - pkt->wqe->state = wqe_state_done; - rxe_sched_task(&qp->comp.task); - } - rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS); goto done; diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c index 6215c6de3a84..67567d62195e 100644 --- a/drivers/infiniband/sw/rxe/rxe_pool.c +++ b/drivers/infiniband/sw/rxe/rxe_pool.c @@ -119,7 +119,7 @@ void rxe_pool_cleanup(struct rxe_pool *pool) int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem, bool sleepable) { - int err; + int err = -EINVAL; gfp_t gfp_flags; if (atomic_inc_return(&pool->num_elem) > pool->max_elem) @@ -147,7 +147,7 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem, err_cnt: atomic_dec(&pool->num_elem); - return -EINVAL; + return err; } void *rxe_pool_get_index(struct rxe_pool *pool, u32 index) diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index e3589c02013e..d2f7b5195c19 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -244,7 +244,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, err = sock_create_kern(&init_net, AF_INET, SOCK_DGRAM, 0, &qp->sk); if (err < 0) return err; - qp->sk->sk->sk_user_data = qp; + qp->sk->sk->sk_user_data = (void *)(uintptr_t)qp->elem.index; /* pick a source UDP port number for this QP based on * the source QPN. this spreads traffic for different QPs @@ -265,8 +265,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, qp->req.opcode = -1; qp->comp.opcode = -1; - rxe_init_task(&qp->req.task, qp, rxe_requester); - rxe_init_task(&qp->comp.task, qp, rxe_completer); + rxe_init_task(&qp->send_task, qp, rxe_sender); qp->qp_timeout_jiffies = 0; /* Can't be set for UD/UC in modify_qp */ if (init->qp_type == IB_QPT_RC) { @@ -337,7 +336,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp, return err; } - rxe_init_task(&qp->resp.task, qp, rxe_responder); + rxe_init_task(&qp->recv_task, qp, rxe_receiver); qp->resp.opcode = OPCODE_NONE; qp->resp.msn = 0; @@ -514,14 +513,12 @@ err1: static void rxe_qp_reset(struct rxe_qp *qp) { /* stop tasks from running */ - rxe_disable_task(&qp->resp.task); - rxe_disable_task(&qp->comp.task); - rxe_disable_task(&qp->req.task); + rxe_disable_task(&qp->recv_task); + rxe_disable_task(&qp->send_task); /* drain work and packet queuesc */ - rxe_requester(qp); - rxe_completer(qp); - rxe_responder(qp); + rxe_sender(qp); + rxe_receiver(qp); if (qp->rq.queue) rxe_queue_reset(qp->rq.queue); @@ -548,9 +545,8 @@ static void rxe_qp_reset(struct rxe_qp *qp) cleanup_rd_atomic_resources(qp); /* reenable tasks */ - rxe_enable_task(&qp->resp.task); - rxe_enable_task(&qp->comp.task); - rxe_enable_task(&qp->req.task); + rxe_enable_task(&qp->recv_task); + rxe_enable_task(&qp->send_task); } /* move the qp to the error state */ @@ -562,9 +558,8 @@ void rxe_qp_error(struct rxe_qp *qp) qp->attr.qp_state = IB_QPS_ERR; /* drain work and packet queues */ - rxe_sched_task(&qp->resp.task); - rxe_sched_task(&qp->comp.task); - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->recv_task); + rxe_sched_task(&qp->send_task); spin_unlock_irqrestore(&qp->state_lock, flags); } @@ -575,8 +570,7 @@ static void rxe_qp_sqd(struct rxe_qp *qp, struct ib_qp_attr *attr, spin_lock_irqsave(&qp->state_lock, flags); qp->attr.sq_draining = 1; - rxe_sched_task(&qp->comp.task); - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); spin_unlock_irqrestore(&qp->state_lock, flags); } @@ -821,19 +815,15 @@ static void rxe_qp_do_cleanup(struct work_struct *work) del_timer_sync(&qp->rnr_nak_timer); } - if (qp->resp.task.func) - rxe_cleanup_task(&qp->resp.task); + if (qp->recv_task.func) + rxe_cleanup_task(&qp->recv_task); - if (qp->req.task.func) - rxe_cleanup_task(&qp->req.task); - - if (qp->comp.task.func) - rxe_cleanup_task(&qp->comp.task); + if (qp->send_task.func) + rxe_cleanup_task(&qp->send_task); /* flush out any receive wr's or pending requests */ - rxe_requester(qp); - rxe_completer(qp); - rxe_responder(qp); + rxe_sender(qp); + rxe_receiver(qp); if (qp->sq.queue) rxe_queue_cleanup(qp->sq.queue); diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index d8c41fd626a9..cd14c4c2dff9 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -108,7 +108,7 @@ void rnr_nak_timer(struct timer_list *t) /* request a send queue retry */ qp->req.need_retry = 1; qp->req.wait_for_rnr_timer = 0; - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); } @@ -545,6 +545,8 @@ static void update_wqe_state(struct rxe_qp *qp, if (pkt->mask & RXE_END_MASK) { if (qp_type(qp) == IB_QPT_RC) wqe->state = wqe_state_pending; + else + wqe->state = wqe_state_done; } else { wqe->state = wqe_state_processing; } @@ -573,30 +575,6 @@ static void update_wqe_psn(struct rxe_qp *qp, qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK; } -static void save_state(struct rxe_send_wqe *wqe, - struct rxe_qp *qp, - struct rxe_send_wqe *rollback_wqe, - u32 *rollback_psn) -{ - rollback_wqe->state = wqe->state; - rollback_wqe->first_psn = wqe->first_psn; - rollback_wqe->last_psn = wqe->last_psn; - rollback_wqe->dma = wqe->dma; - *rollback_psn = qp->req.psn; -} - -static void rollback_state(struct rxe_send_wqe *wqe, - struct rxe_qp *qp, - struct rxe_send_wqe *rollback_wqe, - u32 rollback_psn) -{ - wqe->state = rollback_wqe->state; - wqe->first_psn = rollback_wqe->first_psn; - wqe->last_psn = rollback_wqe->last_psn; - wqe->dma = rollback_wqe->dma; - qp->req.psn = rollback_psn; -} - static void update_state(struct rxe_qp *qp, struct rxe_pkt_info *pkt) { qp->req.opcode = pkt->opcode; @@ -655,12 +633,6 @@ static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe) wqe->status = IB_WC_SUCCESS; qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index); - /* There is no ack coming for local work requests - * which can lead to a deadlock. So go ahead and complete - * it now. - */ - rxe_sched_task(&qp->comp.task); - return 0; } @@ -676,8 +648,6 @@ int rxe_requester(struct rxe_qp *qp) int opcode; int err; int ret; - struct rxe_send_wqe rollback_wqe; - u32 rollback_psn; struct rxe_queue *q = qp->sq.queue; struct rxe_ah *ah; struct rxe_av *av; @@ -786,7 +756,6 @@ int rxe_requester(struct rxe_qp *qp) qp->req.wqe_index); wqe->state = wqe_state_done; wqe->status = IB_WC_SUCCESS; - rxe_sched_task(&qp->comp.task); goto done; } payload = mtu; @@ -799,9 +768,6 @@ int rxe_requester(struct rxe_qp *qp) pkt.mask = rxe_opcode[opcode].mask; pkt.wqe = wqe; - /* save wqe state before we build and send packet */ - save_state(wqe, qp, &rollback_wqe, &rollback_psn); - av = rxe_get_av(&pkt, &ah); if (unlikely(!av)) { rxe_dbg_qp(qp, "Failed no address vector\n"); @@ -834,31 +800,14 @@ int rxe_requester(struct rxe_qp *qp) if (ah) rxe_put(ah); - /* update wqe state as though we had sent it */ - update_wqe_state(qp, wqe, &pkt); - update_wqe_psn(qp, wqe, &pkt, payload); - err = rxe_xmit_packet(qp, &pkt, skb); if (err) { - if (err != -EAGAIN) { - wqe->status = IB_WC_LOC_QP_OP_ERR; - goto err; - } - - /* the packet was dropped so reset wqe to the state - * before we sent it so we can try to resend - */ - rollback_state(wqe, qp, &rollback_wqe, rollback_psn); - - /* force a delay until the dropped packet is freed and - * the send queue is drained below the low water mark - */ - qp->need_req_skb = 1; - - rxe_sched_task(&qp->req.task); - goto exit; + wqe->status = IB_WC_LOC_QP_OP_ERR; + goto err; } + update_wqe_state(qp, wqe, &pkt); + update_wqe_psn(qp, wqe, &pkt, payload); update_state(qp, &pkt); /* A non-zero return value will cause rxe_do_task to @@ -878,3 +827,20 @@ exit: out: return ret; } + +int rxe_sender(struct rxe_qp *qp) +{ + int req_ret; + int comp_ret; + + /* process the send queue */ + req_ret = rxe_requester(qp); + + /* process the response queue */ + comp_ret = rxe_completer(qp); + + /* exit the task loop if both requester and completer + * are ready + */ + return (req_ret && comp_ret) ? -EAGAIN : 0; +} diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 963382f625d7..c6a7fa3054fa 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -49,18 +49,8 @@ static char *resp_state_name[] = { /* rxe_recv calls here to add a request packet to the input queue */ void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) { - int must_sched; - struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); - skb_queue_tail(&qp->req_pkts, skb); - - must_sched = (pkt->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST) || - (skb_queue_len(&qp->req_pkts) > 1); - - if (must_sched) - rxe_sched_task(&qp->resp.task); - else - rxe_run_task(&qp->resp.task); + rxe_sched_task(&qp->recv_task); } static inline enum resp_states get_req(struct rxe_qp *qp, @@ -1485,7 +1475,7 @@ static void flush_recv_queue(struct rxe_qp *qp, bool notify) qp->resp.wqe = NULL; } -int rxe_responder(struct rxe_qp *qp) +int rxe_receiver(struct rxe_qp *qp) { struct rxe_dev *rxe = to_rdev(qp->ibqp.device); enum resp_states state; diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 614581989b38..c7d4d8ab5a09 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -888,6 +888,7 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, { int err = 0; unsigned long flags; + int good = 0; spin_lock_irqsave(&qp->sq.sq_lock, flags); while (ibwr) { @@ -895,18 +896,16 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, if (err) { *bad_wr = ibwr; break; + } else { + good++; } ibwr = ibwr->next; } spin_unlock_irqrestore(&qp->sq.sq_lock, flags); - if (!err) - rxe_sched_task(&qp->req.task); - - spin_lock_irqsave(&qp->state_lock, flags); - if (qp_state(qp) == IB_QPS_ERR) - rxe_sched_task(&qp->comp.task); - spin_unlock_irqrestore(&qp->state_lock, flags); + /* kickoff processing of any posted wqes */ + if (good) + rxe_sched_task(&qp->send_task); return err; } @@ -936,7 +935,7 @@ static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, if (qp->is_user) { /* Utilize process context to do protocol processing */ - rxe_run_task(&qp->req.task); + rxe_sched_task(&qp->send_task); } else { err = rxe_post_send_kernel(qp, wr, bad_wr); if (err) @@ -1046,7 +1045,7 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, spin_lock_irqsave(&qp->state_lock, flags); if (qp_state(qp) == IB_QPS_ERR) - rxe_sched_task(&qp->resp.task); + rxe_sched_task(&qp->recv_task); spin_unlock_irqrestore(&qp->state_lock, flags); return err; diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index ccb9d19ffe8a..3c1354f82283 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -113,7 +113,7 @@ struct rxe_req_info { int need_retry; int wait_for_rnr_timer; int noack_pkts; - struct rxe_task task; + int again; }; struct rxe_comp_info { @@ -124,7 +124,6 @@ struct rxe_comp_info { int started_retry; u32 retry_cnt; u32 rnr_retry; - struct rxe_task task; }; enum rdatm_res_state { @@ -196,7 +195,6 @@ struct rxe_resp_info { unsigned int res_head; unsigned int res_tail; struct resp_res *res; - struct rxe_task task; }; struct rxe_qp { @@ -229,6 +227,9 @@ struct rxe_qp { struct sk_buff_head req_pkts; struct sk_buff_head resp_pkts; + struct rxe_task send_task; + struct rxe_task recv_task; + struct rxe_req_info req; struct rxe_comp_info comp; struct rxe_resp_info resp; diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index 32f6a28b02b1..4e31bb0b6466 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -329,8 +329,7 @@ static struct net_device *ipoib_get_master_net_dev(struct net_device *dev) rcu_read_lock(); master = netdev_master_upper_dev_get_rcu(dev); - if (master) - dev_hold(master); + dev_hold(master); rcu_read_unlock(); if (master) diff --git a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c index 4bd161e86f8d..562df2b3ef18 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c @@ -184,8 +184,12 @@ int ipoib_vlan_add(struct net_device *pdev, unsigned short pkey) ppriv = ipoib_priv(pdev); - snprintf(intf_name, sizeof(intf_name), "%s.%04x", - ppriv->dev->name, pkey); + /* If you increase IFNAMSIZ, update snprintf below + * to allow longer names. + */ + BUILD_BUG_ON(IFNAMSIZ != 16); + snprintf(intf_name, sizeof(intf_name), "%.10s.%04x", ppriv->dev->name, + pkey); ndev = ipoib_intf_alloc(ppriv->ca, ppriv->port, intf_name); if (IS_ERR(ndev)) { diff --git a/drivers/input/serio/ambakmi.c b/drivers/input/serio/ambakmi.c index 8fbfa448be4a..496bb7a312d2 100644 --- a/drivers/input/serio/ambakmi.c +++ b/drivers/input/serio/ambakmi.c @@ -195,7 +195,6 @@ MODULE_DEVICE_TABLE(amba, amba_kmi_idtable); static struct amba_driver ambakmi_driver = { .drv = { .name = "kmi-pl050", - .owner = THIS_MODULE, .pm = pm_sleep_ptr(&amba_kmi_dev_pm_ops), }, .id_table = amba_kmi_idtable, diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 0af39bbbe3a3..c04584be3089 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -376,13 +376,17 @@ config ARM_SMMU_QCOM config ARM_SMMU_QCOM_DEBUG bool "ARM SMMU QCOM implementation defined debug support" - depends on ARM_SMMU_QCOM + depends on ARM_SMMU_QCOM=y help Support for implementation specific debug features in ARM SMMU - hardware found in QTI platforms. + hardware found in QTI platforms. This include support for + the Translation Buffer Units (TBU) that can be used to obtain + additional information when debugging memory management issues + like context faults. - Say Y here to enable debug for issues such as TLB sync timeouts - which requires implementation defined register dumps. + Say Y here to enable debug for issues such as context faults + or TLB sync timeouts which requires implementation defined + register dumps. config ARM_SMMU_V3 tristate "ARM Ltd. System MMU Version 3 (SMMUv3) Support" @@ -397,9 +401,9 @@ config ARM_SMMU_V3 Say Y here if your system includes an IOMMU device implementing the ARM SMMUv3 architecture. +if ARM_SMMU_V3 config ARM_SMMU_V3_SVA bool "Shared Virtual Addressing support for the ARM SMMUv3" - depends on ARM_SMMU_V3 select IOMMU_SVA select IOMMU_IOPF select MMU_NOTIFIER @@ -410,6 +414,17 @@ config ARM_SMMU_V3_SVA Say Y here if your system supports SVA extensions such as PCIe PASID and PRI. +config ARM_SMMU_V3_KUNIT_TEST + tristate "KUnit tests for arm-smmu-v3 driver" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on ARM_SMMU_V3_SVA + default KUNIT_ALL_TESTS + help + Enable this option to unit-test arm-smmu-v3 driver functions. + + If unsure, say N. +endif + config S390_IOMMU def_bool y if S390 && PCI depends on S390 && PCI diff --git a/drivers/iommu/amd/Kconfig b/drivers/iommu/amd/Kconfig index 443b2c13c37b..994063e5586f 100644 --- a/drivers/iommu/amd/Kconfig +++ b/drivers/iommu/amd/Kconfig @@ -7,9 +7,12 @@ config AMD_IOMMU select PCI_ATS select PCI_PRI select PCI_PASID + select MMU_NOTIFIER select IOMMU_API select IOMMU_IOVA select IOMMU_IO_PGTABLE + select IOMMU_SVA + select IOMMU_IOPF select IOMMUFD_DRIVER if IOMMUFD depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE help diff --git a/drivers/iommu/amd/Makefile b/drivers/iommu/amd/Makefile index f454fbb1569e..9de33b2d42f5 100644 --- a/drivers/iommu/amd/Makefile +++ b/drivers/iommu/amd/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o +obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o pasid.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index f482aab420f7..2fde1302a584 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -17,10 +17,16 @@ irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data); irqreturn_t amd_iommu_int_thread_galog(int irq, void *data); irqreturn_t amd_iommu_int_handler(int irq, void *data); void amd_iommu_apply_erratum_63(struct amd_iommu *iommu, u16 devid); +void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, + u8 cntrl_intr, u8 cntrl_log, + u32 status_run_mask, u32 status_overflow_mask); void amd_iommu_restart_event_logging(struct amd_iommu *iommu); void amd_iommu_restart_ga_log(struct amd_iommu *iommu); void amd_iommu_restart_ppr_log(struct amd_iommu *iommu); void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid); +void iommu_feature_enable(struct amd_iommu *iommu, u8 bit); +void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, + gfp_t gfp, size_t size); #ifdef CONFIG_AMD_IOMMU_DEBUGFS void amd_iommu_debugfs_setup(struct amd_iommu *iommu); @@ -33,22 +39,47 @@ int amd_iommu_prepare(void); int amd_iommu_enable(void); void amd_iommu_disable(void); int amd_iommu_reenable(int mode); -int amd_iommu_enable_faulting(void); +int amd_iommu_enable_faulting(unsigned int cpu); extern int amd_iommu_guest_ir; extern enum io_pgtable_fmt amd_iommu_pgtable; extern int amd_iommu_gpt_level; -bool amd_iommu_v2_supported(void); - -/* Device capabilities */ -int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev); -void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev); +/* Protection domain ops */ +struct protection_domain *protection_domain_alloc(unsigned int type); +void protection_domain_free(struct protection_domain *domain); +struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev, + struct mm_struct *mm); +void amd_iommu_domain_free(struct iommu_domain *dom); +int iommu_sva_set_dev_pasid(struct iommu_domain *domain, + struct device *dev, ioasid_t pasid); +void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid, + struct iommu_domain *domain); + +/* SVA/PASID */ +bool amd_iommu_pasid_supported(void); + +/* IOPF */ +int amd_iommu_iopf_init(struct amd_iommu *iommu); +void amd_iommu_iopf_uninit(struct amd_iommu *iommu); +void amd_iommu_page_response(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *resp); +int amd_iommu_iopf_add_device(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data); +void amd_iommu_iopf_remove_device(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data); /* GCR3 setup */ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, unsigned long gcr3); int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid); +/* PPR */ +int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu); +void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu); +void amd_iommu_enable_ppr_log(struct amd_iommu *iommu); +void amd_iommu_poll_ppr_log(struct amd_iommu *iommu); +int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag); + /* * This function flushes all internal caches of * the IOMMU used by this driver. @@ -56,6 +87,7 @@ int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid); void amd_iommu_flush_all_caches(struct amd_iommu *iommu); void amd_iommu_update_and_flush_device_table(struct protection_domain *domain); void amd_iommu_domain_update(struct protection_domain *domain); +void amd_iommu_dev_update_dte(struct iommu_dev_data *dev_data, bool set); void amd_iommu_domain_flush_complete(struct protection_domain *domain); void amd_iommu_domain_flush_pages(struct protection_domain *domain, u64 address, size_t size); @@ -73,9 +105,6 @@ static inline int amd_iommu_create_irq_domain(struct amd_iommu *iommu) } #endif -int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, - int status, int tag); - static inline bool is_rd890_iommu(struct pci_dev *pdev) { return (pdev->vendor == PCI_VENDOR_ID_ATI) && @@ -134,14 +163,6 @@ static inline int get_pci_sbdf_id(struct pci_dev *pdev) return PCI_SEG_DEVID_TO_SBDF(seg, devid); } -static inline void *alloc_pgtable_page(int nid, gfp_t gfp) -{ - struct page *page; - - page = alloc_pages_node(nid, gfp | __GFP_ZERO, 0); - return page ? page_address(page) : NULL; -} - /* * This must be called after device probe completes. During probe * use rlookup_amd_iommu() get the iommu. @@ -157,6 +178,11 @@ static inline struct amd_iommu *get_amd_iommu_from_dev_data(struct iommu_dev_dat return iommu_get_iommu_dev(dev_data->dev, struct amd_iommu, iommu); } +static inline struct protection_domain *to_pdomain(struct iommu_domain *dom) +{ + return container_of(dom, struct protection_domain, domain); +} + bool translation_pre_enabled(struct amd_iommu *iommu); bool amd_iommu_is_attach_deferred(struct device *dev); int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index d1fed5fc219b..2b76b5dedc1d 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -8,7 +8,9 @@ #ifndef _ASM_X86_AMD_IOMMU_TYPES_H #define _ASM_X86_AMD_IOMMU_TYPES_H +#include <linux/iommu.h> #include <linux/types.h> +#include <linux/mmu_notifier.h> #include <linux/mutex.h> #include <linux/msi.h> #include <linux/list.h> @@ -251,6 +253,14 @@ #define PPR_ENTRY_SIZE 16 #define PPR_LOG_SIZE (PPR_ENTRY_SIZE * PPR_LOG_ENTRIES) +/* PAGE_SERVICE_REQUEST PPR Log Buffer Entry flags */ +#define PPR_FLAG_EXEC 0x002 /* Execute permission requested */ +#define PPR_FLAG_READ 0x004 /* Read permission requested */ +#define PPR_FLAG_WRITE 0x020 /* Write permission requested */ +#define PPR_FLAG_US 0x040 /* 1: User, 0: Supervisor */ +#define PPR_FLAG_RVSD 0x080 /* Reserved bit not zero */ +#define PPR_FLAG_GN 0x100 /* GVA and PASID is valid */ + #define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL) #define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL) #define PPR_DEVID(x) ((x) & 0xffffULL) @@ -503,6 +513,11 @@ extern struct kmem_cache *amd_iommu_irq_cache; list_for_each_entry((iommu), &amd_iommu_list, list) #define for_each_iommu_safe(iommu, next) \ list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list) +/* Making iterating over protection_domain->dev_data_list easier */ +#define for_each_pdom_dev_data(pdom_dev_data, pdom) \ + list_for_each_entry(pdom_dev_data, &pdom->dev_data_list, list) +#define for_each_pdom_dev_data_safe(pdom_dev_data, next, pdom) \ + list_for_each_entry_safe((pdom_dev_data), (next), &pdom->dev_data_list, list) struct amd_iommu; struct iommu_domain; @@ -544,6 +559,16 @@ enum protection_domain_mode { PD_MODE_V2, }; +/* Track dev_data/PASID list for the protection domain */ +struct pdom_dev_data { + /* Points to attached device data */ + struct iommu_dev_data *dev_data; + /* PASID attached to the protection domain */ + ioasid_t pasid; + /* For protection_domain->dev_data_list */ + struct list_head list; +}; + /* * This structure contains generic data for IOMMU protection domains * independent of their use. @@ -560,6 +585,9 @@ struct protection_domain { bool dirty_tracking; /* dirty tracking is enabled in the domain */ unsigned dev_cnt; /* devices assigned to this domain */ unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ + + struct mmu_notifier mn; /* mmu notifier for the SVA domain */ + struct list_head dev_data_list; /* List of pdom_dev_data */ }; /* @@ -762,6 +790,10 @@ struct amd_iommu { /* DebugFS Info */ struct dentry *debugfs; #endif + + /* IOPF support */ + struct iopf_queue *iopf_queue; + unsigned char iopfq_name[32]; }; static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev) @@ -813,6 +845,7 @@ struct iommu_dev_data { struct device *dev; u16 devid; /* PCI Device ID */ + u32 max_pasids; /* Max supported PASIDs */ u32 flags; /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */ int ats_qdep; u8 ats_enabled :1; /* ATS state */ diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index ac6754a85f35..a18e74878f68 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -36,6 +36,7 @@ #include "amd_iommu.h" #include "../irq_remapping.h" +#include "../iommu-pages.h" /* * definitions for the ACPI scanning code @@ -419,7 +420,7 @@ static void iommu_set_device_table(struct amd_iommu *iommu) } /* Generic functions to enable/disable certain features of the IOMMU. */ -static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) +void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) { u64 ctrl; @@ -649,8 +650,8 @@ static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_ /* Allocate per PCI segment device table */ static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) { - pci_seg->dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32, - get_order(pci_seg->dev_table_size)); + pci_seg->dev_table = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32, + get_order(pci_seg->dev_table_size)); if (!pci_seg->dev_table) return -ENOMEM; @@ -659,17 +660,16 @@ static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg) { - free_pages((unsigned long)pci_seg->dev_table, - get_order(pci_seg->dev_table_size)); + iommu_free_pages(pci_seg->dev_table, + get_order(pci_seg->dev_table_size)); pci_seg->dev_table = NULL; } /* Allocate per PCI segment IOMMU rlookup table. */ static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg) { - pci_seg->rlookup_table = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, - get_order(pci_seg->rlookup_table_size)); + pci_seg->rlookup_table = iommu_alloc_pages(GFP_KERNEL, + get_order(pci_seg->rlookup_table_size)); if (pci_seg->rlookup_table == NULL) return -ENOMEM; @@ -678,16 +678,15 @@ static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg) static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg) { - free_pages((unsigned long)pci_seg->rlookup_table, - get_order(pci_seg->rlookup_table_size)); + iommu_free_pages(pci_seg->rlookup_table, + get_order(pci_seg->rlookup_table_size)); pci_seg->rlookup_table = NULL; } static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) { - pci_seg->irq_lookup_table = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, - get_order(pci_seg->rlookup_table_size)); + pci_seg->irq_lookup_table = iommu_alloc_pages(GFP_KERNEL, + get_order(pci_seg->rlookup_table_size)); kmemleak_alloc(pci_seg->irq_lookup_table, pci_seg->rlookup_table_size, 1, GFP_KERNEL); if (pci_seg->irq_lookup_table == NULL) @@ -699,8 +698,8 @@ static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_se static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) { kmemleak_free(pci_seg->irq_lookup_table); - free_pages((unsigned long)pci_seg->irq_lookup_table, - get_order(pci_seg->rlookup_table_size)); + iommu_free_pages(pci_seg->irq_lookup_table, + get_order(pci_seg->rlookup_table_size)); pci_seg->irq_lookup_table = NULL; } @@ -708,8 +707,8 @@ static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg) { int i; - pci_seg->alias_table = (void *)__get_free_pages(GFP_KERNEL, - get_order(pci_seg->alias_table_size)); + pci_seg->alias_table = iommu_alloc_pages(GFP_KERNEL, + get_order(pci_seg->alias_table_size)); if (!pci_seg->alias_table) return -ENOMEM; @@ -724,8 +723,8 @@ static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg) static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg) { - free_pages((unsigned long)pci_seg->alias_table, - get_order(pci_seg->alias_table_size)); + iommu_free_pages(pci_seg->alias_table, + get_order(pci_seg->alias_table_size)); pci_seg->alias_table = NULL; } @@ -736,8 +735,8 @@ static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg) */ static int __init alloc_command_buffer(struct amd_iommu *iommu) { - iommu->cmd_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(CMD_BUFFER_SIZE)); + iommu->cmd_buf = iommu_alloc_pages(GFP_KERNEL, + get_order(CMD_BUFFER_SIZE)); return iommu->cmd_buf ? 0 : -ENOMEM; } @@ -746,9 +745,9 @@ static int __init alloc_command_buffer(struct amd_iommu *iommu) * Interrupt handler has processed all pending events and adjusted head * and tail pointer. Reset overflow mask and restart logging again. */ -static void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, - u8 cntrl_intr, u8 cntrl_log, - u32 status_run_mask, u32 status_overflow_mask) +void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, + u8 cntrl_intr, u8 cntrl_log, + u32 status_run_mask, u32 status_overflow_mask) { u32 status; @@ -790,17 +789,6 @@ void amd_iommu_restart_ga_log(struct amd_iommu *iommu) } /* - * This function restarts ppr logging in case the IOMMU experienced - * PPR log overflow. - */ -void amd_iommu_restart_ppr_log(struct amd_iommu *iommu) -{ - amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN, - CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK, - MMIO_STATUS_PPR_OVERFLOW_MASK); -} - -/* * This function resets the command buffer if the IOMMU stopped fetching * commands from it. */ @@ -845,19 +833,19 @@ static void iommu_disable_command_buffer(struct amd_iommu *iommu) static void __init free_command_buffer(struct amd_iommu *iommu) { - free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE)); + iommu_free_pages(iommu->cmd_buf, get_order(CMD_BUFFER_SIZE)); } -static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, - gfp_t gfp, size_t size) +void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp, + size_t size) { int order = get_order(size); - void *buf = (void *)__get_free_pages(gfp, order); + void *buf = iommu_alloc_pages(gfp, order); if (buf && check_feature(FEATURE_SNP) && set_memory_4k((unsigned long)buf, (1 << order))) { - free_pages((unsigned long)buf, order); + iommu_free_pages(buf, order); buf = NULL; } @@ -867,7 +855,7 @@ static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, /* allocates the memory where the IOMMU will log its events to */ static int __init alloc_event_buffer(struct amd_iommu *iommu) { - iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, + iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL, EVT_BUFFER_SIZE); return iommu->evt_buf ? 0 : -ENOMEM; @@ -901,50 +889,14 @@ static void iommu_disable_event_buffer(struct amd_iommu *iommu) static void __init free_event_buffer(struct amd_iommu *iommu) { - free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); -} - -/* allocates the memory where the IOMMU will log its events to */ -static int __init alloc_ppr_log(struct amd_iommu *iommu) -{ - iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, - PPR_LOG_SIZE); - - return iommu->ppr_log ? 0 : -ENOMEM; -} - -static void iommu_enable_ppr_log(struct amd_iommu *iommu) -{ - u64 entry; - - if (iommu->ppr_log == NULL) - return; - - iommu_feature_enable(iommu, CONTROL_PPR_EN); - - entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512; - - memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET, - &entry, sizeof(entry)); - - /* set head and tail to zero manually */ - writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); - - iommu_feature_enable(iommu, CONTROL_PPRLOG_EN); - iommu_feature_enable(iommu, CONTROL_PPRINT_EN); -} - -static void __init free_ppr_log(struct amd_iommu *iommu) -{ - free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE)); + iommu_free_pages(iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); } static void free_ga_log(struct amd_iommu *iommu) { #ifdef CONFIG_IRQ_REMAP - free_pages((unsigned long)iommu->ga_log, get_order(GA_LOG_SIZE)); - free_pages((unsigned long)iommu->ga_log_tail, get_order(8)); + iommu_free_pages(iommu->ga_log, get_order(GA_LOG_SIZE)); + iommu_free_pages(iommu->ga_log_tail, get_order(8)); #endif } @@ -989,13 +941,11 @@ static int iommu_init_ga_log(struct amd_iommu *iommu) if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) return 0; - iommu->ga_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(GA_LOG_SIZE)); + iommu->ga_log = iommu_alloc_pages(GFP_KERNEL, get_order(GA_LOG_SIZE)); if (!iommu->ga_log) goto err_out; - iommu->ga_log_tail = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(8)); + iommu->ga_log_tail = iommu_alloc_pages(GFP_KERNEL, get_order(8)); if (!iommu->ga_log_tail) goto err_out; @@ -1008,7 +958,7 @@ err_out: static int __init alloc_cwwb_sem(struct amd_iommu *iommu) { - iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 1); + iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 1); return iommu->cmd_sem ? 0 : -ENOMEM; } @@ -1016,7 +966,7 @@ static int __init alloc_cwwb_sem(struct amd_iommu *iommu) static void __init free_cwwb_sem(struct amd_iommu *iommu) { if (iommu->cmd_sem) - free_page((unsigned long)iommu->cmd_sem); + iommu_free_page((void *)iommu->cmd_sem); } static void iommu_enable_xt(struct amd_iommu *iommu) @@ -1081,7 +1031,6 @@ static bool __copy_device_table(struct amd_iommu *iommu) u32 lo, hi, devid, old_devtb_size; phys_addr_t old_devtb_phys; u16 dom_id, dte_v, irq_v; - gfp_t gfp_flag; u64 tmp; /* Each IOMMU use separate device table with the same size */ @@ -1115,9 +1064,8 @@ static bool __copy_device_table(struct amd_iommu *iommu) if (!old_devtb) return false; - gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32; - pci_seg->old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag, - get_order(pci_seg->dev_table_size)); + pci_seg->old_dev_tbl_cpy = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32, + get_order(pci_seg->dev_table_size)); if (pci_seg->old_dev_tbl_cpy == NULL) { pr_err("Failed to allocate memory for copying old device table!\n"); memunmap(old_devtb); @@ -1683,9 +1631,10 @@ static void __init free_iommu_one(struct amd_iommu *iommu) free_cwwb_sem(iommu); free_command_buffer(iommu); free_event_buffer(iommu); - free_ppr_log(iommu); + amd_iommu_free_ppr_log(iommu); free_ga_log(iommu); iommu_unmap_mmio_space(iommu); + amd_iommu_iopf_uninit(iommu); } static void __init free_iommu_all(void) @@ -2097,9 +2046,11 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) amd_iommu_max_glx_val = glxval; else amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval); + + iommu_enable_gt(iommu); } - if (check_feature(FEATURE_PPR) && alloc_ppr_log(iommu)) + if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu)) return -ENOMEM; if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) { @@ -2155,6 +2106,16 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) if (ret) return ret; + /* + * Allocate per IOMMU IOPF queue here so that in attach device path, + * PRI capable device can be added to IOPF queue + */ + if (amd_iommu_gt_ppr_supported()) { + ret = amd_iommu_iopf_init(iommu); + if (ret) + return ret; + } + iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL); return pci_enable_device(iommu->dev); @@ -2773,7 +2734,6 @@ static void early_enable_iommu(struct amd_iommu *iommu) iommu_enable_command_buffer(iommu); iommu_enable_event_buffer(iommu); iommu_set_exclusion_range(iommu); - iommu_enable_gt(iommu); iommu_enable_ga(iommu); iommu_enable_xt(iommu); iommu_enable_irtcachedis(iommu); @@ -2805,8 +2765,8 @@ static void early_enable_iommus(void) for_each_pci_segment(pci_seg) { if (pci_seg->old_dev_tbl_cpy != NULL) { - free_pages((unsigned long)pci_seg->old_dev_tbl_cpy, - get_order(pci_seg->dev_table_size)); + iommu_free_pages(pci_seg->old_dev_tbl_cpy, + get_order(pci_seg->dev_table_size)); pci_seg->old_dev_tbl_cpy = NULL; } } @@ -2819,8 +2779,8 @@ static void early_enable_iommus(void) pr_info("Copied DEV table from previous kernel.\n"); for_each_pci_segment(pci_seg) { - free_pages((unsigned long)pci_seg->dev_table, - get_order(pci_seg->dev_table_size)); + iommu_free_pages(pci_seg->dev_table, + get_order(pci_seg->dev_table_size)); pci_seg->dev_table = pci_seg->old_dev_tbl_cpy; } @@ -2830,7 +2790,6 @@ static void early_enable_iommus(void) iommu_disable_irtcachedis(iommu); iommu_enable_command_buffer(iommu); iommu_enable_event_buffer(iommu); - iommu_enable_gt(iommu); iommu_enable_ga(iommu); iommu_enable_xt(iommu); iommu_enable_irtcachedis(iommu); @@ -2840,12 +2799,15 @@ static void early_enable_iommus(void) } } -static void enable_iommus_v2(void) +static void enable_iommus_ppr(void) { struct amd_iommu *iommu; + if (!amd_iommu_gt_ppr_supported()) + return; + for_each_iommu(iommu) - iommu_enable_ppr_log(iommu); + amd_iommu_enable_ppr_log(iommu); } static void enable_iommus_vapic(void) @@ -3022,8 +2984,8 @@ static bool __init check_ioapic_information(void) static void __init free_dma_resources(void) { - free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, - get_order(MAX_DOMAIN_ID/8)); + iommu_free_pages(amd_iommu_pd_alloc_bitmap, + get_order(MAX_DOMAIN_ID / 8)); amd_iommu_pd_alloc_bitmap = NULL; free_unity_maps(); @@ -3095,9 +3057,8 @@ static int __init early_amd_iommu_init(void) /* Device table - directly used by all IOMMUs */ ret = -ENOMEM; - amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, - get_order(MAX_DOMAIN_ID/8)); + amd_iommu_pd_alloc_bitmap = iommu_alloc_pages(GFP_KERNEL, + get_order(MAX_DOMAIN_ID / 8)); if (amd_iommu_pd_alloc_bitmap == NULL) goto out; @@ -3181,7 +3142,7 @@ static int amd_iommu_enable_interrupts(void) * PPR and GA log interrupt for all IOMMUs. */ enable_iommus_vapic(); - enable_iommus_v2(); + enable_iommus_ppr(); out: return ret; @@ -3392,7 +3353,7 @@ int amd_iommu_reenable(int mode) return 0; } -int __init amd_iommu_enable_faulting(void) +int __init amd_iommu_enable_faulting(unsigned int cpu) { /* We enable MSI later when PCI is initialized */ return 0; @@ -3690,7 +3651,7 @@ __setup("ivrs_ioapic", parse_ivrs_ioapic); __setup("ivrs_hpet", parse_ivrs_hpet); __setup("ivrs_acpihid", parse_ivrs_acpihid); -bool amd_iommu_v2_supported(void) +bool amd_iommu_pasid_supported(void) { /* CPU page table size should match IOMMU guest page table size */ if (cpu_feature_enabled(X86_FEATURE_LA57) && diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index 2a0d1e97e52f..9d9a7fde59e7 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -22,6 +22,7 @@ #include "amd_iommu_types.h" #include "amd_iommu.h" +#include "../iommu-pages.h" static void v1_tlb_flush_all(void *cookie) { @@ -156,7 +157,7 @@ static bool increase_address_space(struct protection_domain *domain, bool ret = true; u64 *pte; - pte = alloc_pgtable_page(domain->nid, gfp); + pte = iommu_alloc_page_node(domain->nid, gfp); if (!pte) return false; @@ -187,7 +188,7 @@ static bool increase_address_space(struct protection_domain *domain, out: spin_unlock_irqrestore(&domain->lock, flags); - free_page((unsigned long)pte); + iommu_free_page(pte); return ret; } @@ -250,7 +251,7 @@ static u64 *alloc_pte(struct protection_domain *domain, if (!IOMMU_PTE_PRESENT(__pte) || pte_level == PAGE_MODE_NONE) { - page = alloc_pgtable_page(domain->nid, gfp); + page = iommu_alloc_page_node(domain->nid, gfp); if (!page) return NULL; @@ -259,7 +260,7 @@ static u64 *alloc_pte(struct protection_domain *domain, /* pte could have been changed somewhere. */ if (!try_cmpxchg64(pte, &__pte, __npte)) - free_page((unsigned long)page); + iommu_free_page(page); else if (IOMMU_PTE_PRESENT(__pte)) *updated = true; @@ -431,7 +432,7 @@ out: } /* Everything flushed out, free pages now */ - put_pages_list(&freelist); + iommu_put_pages_list(&freelist); return ret; } @@ -580,7 +581,7 @@ static void v1_free_pgtable(struct io_pgtable *iop) /* Make changes visible to IOMMUs */ amd_iommu_domain_update(dom); - put_pages_list(&freelist); + iommu_put_pages_list(&freelist); } static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c index 93489d2db4e8..78ac37c5ccc1 100644 --- a/drivers/iommu/amd/io_pgtable_v2.c +++ b/drivers/iommu/amd/io_pgtable_v2.c @@ -18,6 +18,7 @@ #include "amd_iommu_types.h" #include "amd_iommu.h" +#include "../iommu-pages.h" #define IOMMU_PAGE_PRESENT BIT_ULL(0) /* Is present */ #define IOMMU_PAGE_RW BIT_ULL(1) /* Writeable */ @@ -99,11 +100,6 @@ static inline int page_size_to_level(u64 pg_size) return PAGE_MODE_1_LEVEL; } -static inline void free_pgtable_page(u64 *pt) -{ - free_page((unsigned long)pt); -} - static void free_pgtable(u64 *pt, int level) { u64 *p; @@ -125,10 +121,10 @@ static void free_pgtable(u64 *pt, int level) if (level > 2) free_pgtable(p, level - 1); else - free_pgtable_page(p); + iommu_free_page(p); } - free_pgtable_page(pt); + iommu_free_page(pt); } /* Allocate page table */ @@ -156,14 +152,14 @@ static u64 *v2_alloc_pte(int nid, u64 *pgd, unsigned long iova, } if (!IOMMU_PTE_PRESENT(__pte)) { - page = alloc_pgtable_page(nid, gfp); + page = iommu_alloc_page_node(nid, gfp); if (!page) return NULL; __npte = set_pgtable_attr(page); /* pte could have been changed somewhere. */ if (cmpxchg64(pte, __pte, __npte) != __pte) - free_pgtable_page(page); + iommu_free_page(page); else if (IOMMU_PTE_PRESENT(__pte)) *updated = true; @@ -185,7 +181,7 @@ static u64 *v2_alloc_pte(int nid, u64 *pgd, unsigned long iova, if (pg_size == IOMMU_PAGE_SIZE_1G) free_pgtable(__pte, end_level - 1); else if (pg_size == IOMMU_PAGE_SIZE_2M) - free_pgtable_page(__pte); + iommu_free_page(__pte); } return pte; @@ -366,7 +362,7 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo struct protection_domain *pdom = (struct protection_domain *)cookie; int ias = IOMMU_IN_ADDR_BIT_SIZE; - pgtable->pgd = alloc_pgtable_page(pdom->nid, GFP_ATOMIC); + pgtable->pgd = iommu_alloc_page_node(pdom->nid, GFP_ATOMIC); if (!pgtable->pgd) return NULL; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index fb727f5b0b82..52d83730a22a 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -42,6 +42,7 @@ #include "amd_iommu.h" #include "../dma-iommu.h" #include "../irq_remapping.h" +#include "../iommu-pages.h" #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) @@ -89,6 +90,21 @@ static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom) return (pdom && (pdom->pd_mode == PD_MODE_V2)); } +static inline bool pdom_is_in_pt_mode(struct protection_domain *pdom) +{ + return (pdom->domain.type == IOMMU_DOMAIN_IDENTITY); +} + +/* + * We cannot support PASID w/ existing v1 page table in the same domain + * since it will be nested. However, existing domain w/ v2 page table + * or passthrough mode can be used for PASID. + */ +static inline bool pdom_is_sva_capable(struct protection_domain *pdom) +{ + return pdom_is_v2_pgtbl_mode(pdom) || pdom_is_in_pt_mode(pdom); +} + static inline int get_acpihid_device_id(struct device *dev, struct acpihid_map_entry **entry) { @@ -179,11 +195,6 @@ static struct amd_iommu *rlookup_amd_iommu(struct device *dev) return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid)); } -static struct protection_domain *to_pdomain(struct iommu_domain *dom) -{ - return container_of(dom, struct protection_domain, domain); -} - static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid) { struct iommu_dev_data *dev_data; @@ -384,7 +395,7 @@ static inline void pdev_disable_cap_ats(struct pci_dev *pdev) } } -int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev) +static inline int pdev_enable_cap_pri(struct pci_dev *pdev) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); int ret = -EINVAL; @@ -392,6 +403,9 @@ int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev) if (dev_data->pri_enabled) return 0; + if (!dev_data->ats_enabled) + return 0; + if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) { /* * First reset the PRI state of the device. @@ -408,7 +422,7 @@ int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev) return ret; } -void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev) +static inline void pdev_disable_cap_pri(struct pci_dev *pdev) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); @@ -450,15 +464,14 @@ static void pdev_enable_caps(struct pci_dev *pdev) { pdev_enable_cap_ats(pdev); pdev_enable_cap_pasid(pdev); - amd_iommu_pdev_enable_cap_pri(pdev); - + pdev_enable_cap_pri(pdev); } static void pdev_disable_caps(struct pci_dev *pdev) { pdev_disable_cap_ats(pdev); pdev_disable_cap_pasid(pdev); - amd_iommu_pdev_disable_cap_pri(pdev); + pdev_disable_cap_pri(pdev); } /* @@ -818,59 +831,6 @@ static void iommu_poll_events(struct amd_iommu *iommu) writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); } -static void iommu_poll_ppr_log(struct amd_iommu *iommu) -{ - u32 head, tail; - - if (iommu->ppr_log == NULL) - return; - - head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); - - while (head != tail) { - volatile u64 *raw; - u64 entry[2]; - int i; - - raw = (u64 *)(iommu->ppr_log + head); - - /* - * Hardware bug: Interrupt may arrive before the entry is - * written to memory. If this happens we need to wait for the - * entry to arrive. - */ - for (i = 0; i < LOOP_TIMEOUT; ++i) { - if (PPR_REQ_TYPE(raw[0]) != 0) - break; - udelay(1); - } - - /* Avoid memcpy function-call overhead */ - entry[0] = raw[0]; - entry[1] = raw[1]; - - /* - * To detect the hardware errata 733 we need to clear the - * entry back to zero. This issue does not exist on SNP - * enabled system. Also this buffer is not writeable on - * SNP enabled system. - */ - if (!amd_iommu_snp_en) - raw[0] = raw[1] = 0UL; - - /* Update head pointer of hardware ring-buffer */ - head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE; - writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - - /* TODO: PPR Handler will be added when we add IOPF support */ - - /* Refresh ring-buffer information */ - head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); - } -} - #ifdef CONFIG_IRQ_REMAP static int (*iommu_ga_log_notifier)(u32); @@ -991,7 +951,7 @@ irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data) { amd_iommu_handle_irq(data, "PPR", MMIO_STATUS_PPR_INT_MASK, MMIO_STATUS_PPR_OVERFLOW_MASK, - iommu_poll_ppr_log, amd_iommu_restart_ppr_log); + amd_iommu_poll_ppr_log, amd_iommu_restart_ppr_log); return IRQ_HANDLED; } @@ -1664,15 +1624,14 @@ void amd_iommu_domain_update(struct protection_domain *domain) amd_iommu_domain_flush_all(domain); } -int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, - int status, int tag) +int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag) { struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct iommu_cmd cmd; - dev_data = dev_iommu_priv_get(&pdev->dev); - iommu = get_amd_iommu_from_dev(&pdev->dev); + dev_data = dev_iommu_priv_get(dev); + iommu = get_amd_iommu_from_dev(dev); build_complete_ppr(&cmd, dev_data->devid, pasid, status, tag, dev_data->pri_tlp); @@ -1728,7 +1687,7 @@ static void free_gcr3_tbl_level1(u64 *tbl) ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK); - free_page((unsigned long)ptr); + iommu_free_page(ptr); } } @@ -1761,7 +1720,7 @@ static void free_gcr3_table(struct gcr3_tbl_info *gcr3_info) /* Free per device domain ID */ domain_id_free(gcr3_info->domid); - free_page((unsigned long)gcr3_info->gcr3_tbl); + iommu_free_page(gcr3_info->gcr3_tbl); gcr3_info->gcr3_tbl = NULL; } @@ -1796,7 +1755,7 @@ static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info, /* Allocate per device domain ID */ gcr3_info->domid = domain_id_alloc(); - gcr3_info->gcr3_tbl = alloc_pgtable_page(nid, GFP_ATOMIC); + gcr3_info->gcr3_tbl = iommu_alloc_page_node(nid, GFP_ATOMIC); if (gcr3_info->gcr3_tbl == NULL) { domain_id_free(gcr3_info->domid); return -ENOMEM; @@ -2002,10 +1961,78 @@ static void clear_dte_entry(struct amd_iommu *iommu, u16 devid) amd_iommu_apply_erratum_63(iommu, devid); } +/* Update and flush DTE for the given device */ +void amd_iommu_dev_update_dte(struct iommu_dev_data *dev_data, bool set) +{ + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); + + if (set) + set_dte_entry(iommu, dev_data); + else + clear_dte_entry(iommu, dev_data->devid); + + clone_aliases(iommu, dev_data->dev); + device_flush_dte(dev_data); + iommu_completion_wait(iommu); +} + +/* + * If domain is SVA capable then initialize GCR3 table. Also if domain is + * in v2 page table mode then update GCR3[0]. + */ +static int init_gcr3_table(struct iommu_dev_data *dev_data, + struct protection_domain *pdom) +{ + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + int max_pasids = dev_data->max_pasids; + int ret = 0; + + /* + * If domain is in pt mode then setup GCR3 table only if device + * is PASID capable + */ + if (pdom_is_in_pt_mode(pdom) && !pdev_pasid_supported(dev_data)) + return ret; + + /* + * By default, setup GCR3 table to support MAX PASIDs + * supported by the device/IOMMU. + */ + ret = setup_gcr3_table(&dev_data->gcr3_info, iommu, + max_pasids > 0 ? max_pasids : 1); + if (ret) + return ret; + + /* Setup GCR3[0] only if domain is setup with v2 page table mode */ + if (!pdom_is_v2_pgtbl_mode(pdom)) + return ret; + + ret = update_gcr3(dev_data, 0, iommu_virt_to_phys(pdom->iop.pgd), true); + if (ret) + free_gcr3_table(&dev_data->gcr3_info); + + return ret; +} + +static void destroy_gcr3_table(struct iommu_dev_data *dev_data, + struct protection_domain *pdom) +{ + struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; + + if (pdom_is_v2_pgtbl_mode(pdom)) + update_gcr3(dev_data, 0, 0, false); + + if (gcr3_info->gcr3_tbl == NULL) + return; + + free_gcr3_table(gcr3_info); +} + static int do_attach(struct iommu_dev_data *dev_data, struct protection_domain *domain) { struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + struct pci_dev *pdev; int ret = 0; /* Update data structures */ @@ -2020,26 +2047,29 @@ static int do_attach(struct iommu_dev_data *dev_data, domain->dev_iommu[iommu->index] += 1; domain->dev_cnt += 1; - /* Init GCR3 table and update device table */ - if (domain->pd_mode == PD_MODE_V2) { - /* By default, setup GCR3 table to support single PASID */ - ret = setup_gcr3_table(&dev_data->gcr3_info, iommu, 1); + pdev = dev_is_pci(dev_data->dev) ? to_pci_dev(dev_data->dev) : NULL; + if (pdom_is_sva_capable(domain)) { + ret = init_gcr3_table(dev_data, domain); if (ret) return ret; - ret = update_gcr3(dev_data, 0, - iommu_virt_to_phys(domain->iop.pgd), true); - if (ret) { - free_gcr3_table(&dev_data->gcr3_info); - return ret; + if (pdev) { + pdev_enable_caps(pdev); + + /* + * Device can continue to function even if IOPF + * enablement failed. Hence in error path just + * disable device PRI support. + */ + if (amd_iommu_iopf_add_device(iommu, dev_data)) + pdev_disable_cap_pri(pdev); } + } else if (pdev) { + pdev_enable_cap_ats(pdev); } /* Update device table */ - set_dte_entry(iommu, dev_data); - clone_aliases(iommu, dev_data->dev); - - device_flush_dte(dev_data); + amd_iommu_dev_update_dte(dev_data, true); return ret; } @@ -2050,19 +2080,15 @@ static void do_detach(struct iommu_dev_data *dev_data) struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); /* Clear GCR3 table */ - if (domain->pd_mode == PD_MODE_V2) { - update_gcr3(dev_data, 0, 0, false); - free_gcr3_table(&dev_data->gcr3_info); - } + if (pdom_is_sva_capable(domain)) + destroy_gcr3_table(dev_data, domain); /* Update data structures */ dev_data->domain = NULL; list_del(&dev_data->list); - clear_dte_entry(iommu, dev_data->devid); - clone_aliases(iommu, dev_data->dev); - /* Flush the DTE entry */ - device_flush_dte(dev_data); + /* Clear DTE and flush the entry */ + amd_iommu_dev_update_dte(dev_data, false); /* Flush IOTLB and wait for the flushes to finish */ amd_iommu_domain_flush_all(domain); @@ -2094,9 +2120,6 @@ static int attach_device(struct device *dev, goto out; } - if (dev_is_pci(dev)) - pdev_enable_caps(to_pci_dev(dev)); - ret = do_attach(dev_data, domain); out: @@ -2112,12 +2135,11 @@ out: */ static void detach_device(struct device *dev) { - struct protection_domain *domain; - struct iommu_dev_data *dev_data; + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + struct protection_domain *domain = dev_data->domain; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); unsigned long flags; - - dev_data = dev_iommu_priv_get(dev); - domain = dev_data->domain; + bool ppr = dev_data->ppr; spin_lock_irqsave(&domain->lock, flags); @@ -2132,8 +2154,19 @@ static void detach_device(struct device *dev) if (WARN_ON(!dev_data->domain)) goto out; + if (ppr) { + iopf_queue_flush_dev(dev); + + /* Updated here so that it gets reflected in DTE */ + dev_data->ppr = false; + } + do_detach(dev_data); + /* Remove IOPF handler */ + if (ppr) + amd_iommu_iopf_remove_device(iommu, dev_data); + if (dev_is_pci(dev)) pdev_disable_caps(to_pci_dev(dev)); @@ -2147,6 +2180,7 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) { struct iommu_device *iommu_dev; struct amd_iommu *iommu; + struct iommu_dev_data *dev_data; int ret; if (!check_device(dev)) @@ -2173,18 +2207,22 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) iommu_dev = &iommu->iommu; } + /* + * If IOMMU and device supports PASID then it will contain max + * supported PASIDs, else it will be zero. + */ + dev_data = dev_iommu_priv_get(dev); + if (amd_iommu_pasid_supported() && dev_is_pci(dev) && + pdev_pasid_supported(dev_data)) { + dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids, + pci_max_pasids(to_pci_dev(dev))); + } + iommu_completion_wait(iommu); return iommu_dev; } -static void amd_iommu_probe_finalize(struct device *dev) -{ - /* Domains are initialized for this device - have a look what we ended up with */ - set_dma_ops(dev, NULL); - iommu_setup_dma_ops(dev, 0, U64_MAX); -} - static void amd_iommu_release_device(struct device *dev) { struct amd_iommu *iommu; @@ -2236,7 +2274,7 @@ static void cleanup_domain(struct protection_domain *domain) WARN_ON(domain->dev_cnt != 0); } -static void protection_domain_free(struct protection_domain *domain) +void protection_domain_free(struct protection_domain *domain) { if (!domain) return; @@ -2245,7 +2283,7 @@ static void protection_domain_free(struct protection_domain *domain) free_io_pgtable_ops(&domain->iop.iop.ops); if (domain->iop.root) - free_page((unsigned long)domain->iop.root); + iommu_free_page(domain->iop.root); if (domain->id) domain_id_free(domain->id); @@ -2260,7 +2298,7 @@ static int protection_domain_init_v1(struct protection_domain *domain, int mode) BUG_ON(mode < PAGE_MODE_NONE || mode > PAGE_MODE_6_LEVEL); if (mode != PAGE_MODE_NONE) { - pt_root = (void *)get_zeroed_page(GFP_KERNEL); + pt_root = iommu_alloc_page(GFP_KERNEL); if (!pt_root) return -ENOMEM; } @@ -2279,7 +2317,7 @@ static int protection_domain_init_v2(struct protection_domain *pdom) return 0; } -static struct protection_domain *protection_domain_alloc(unsigned int type) +struct protection_domain *protection_domain_alloc(unsigned int type) { struct io_pgtable_ops *pgtbl_ops; struct protection_domain *domain; @@ -2296,11 +2334,13 @@ static struct protection_domain *protection_domain_alloc(unsigned int type) spin_lock_init(&domain->lock); INIT_LIST_HEAD(&domain->dev_list); + INIT_LIST_HEAD(&domain->dev_data_list); domain->nid = NUMA_NO_NODE; switch (type) { /* No need to allocate io pgtable ops in passthrough mode */ case IOMMU_DOMAIN_IDENTITY: + case IOMMU_DOMAIN_SVA: return domain; case IOMMU_DOMAIN_DMA: pgtable = amd_iommu_pgtable; @@ -2420,7 +2460,7 @@ amd_iommu_domain_alloc_user(struct device *dev, u32 flags, return do_iommu_domain_alloc(type, dev, flags); } -static void amd_iommu_domain_free(struct iommu_domain *dom) +void amd_iommu_domain_free(struct iommu_domain *dom) { struct protection_domain *domain; unsigned long flags; @@ -2785,18 +2825,54 @@ static const struct iommu_dirty_ops amd_dirty_ops = { .read_and_clear_dirty = amd_iommu_read_and_clear_dirty, }; +static int amd_iommu_dev_enable_feature(struct device *dev, + enum iommu_dev_features feat) +{ + int ret = 0; + + switch (feat) { + case IOMMU_DEV_FEAT_IOPF: + case IOMMU_DEV_FEAT_SVA: + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +static int amd_iommu_dev_disable_feature(struct device *dev, + enum iommu_dev_features feat) +{ + int ret = 0; + + switch (feat) { + case IOMMU_DEV_FEAT_IOPF: + case IOMMU_DEV_FEAT_SVA: + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, .domain_alloc_user = amd_iommu_domain_alloc_user, + .domain_alloc_sva = amd_iommu_domain_alloc_sva, .probe_device = amd_iommu_probe_device, .release_device = amd_iommu_release_device, - .probe_finalize = amd_iommu_probe_finalize, .device_group = amd_iommu_device_group, .get_resv_regions = amd_iommu_get_resv_regions, .is_attach_deferred = amd_iommu_is_attach_deferred, .pgsize_bitmap = AMD_IOMMU_PGSIZES, .def_domain_type = amd_iommu_def_domain_type, + .dev_enable_feat = amd_iommu_dev_enable_feature, + .dev_disable_feat = amd_iommu_dev_disable_feature, + .remove_dev_pasid = amd_iommu_remove_dev_pasid, + .page_response = amd_iommu_page_response, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = amd_iommu_attach_device, .map_pages = amd_iommu_map_pages, @@ -3708,20 +3784,11 @@ static struct irq_chip amd_ir_chip = { }; static const struct msi_parent_ops amdvi_msi_parent_ops = { - .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | - MSI_FLAG_MULTI_PCI_MSI | - MSI_FLAG_PCI_IMS, + .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | MSI_FLAG_MULTI_PCI_MSI, .prefix = "IR-", .init_dev_msi_info = msi_parent_init_dev_msi_info, }; -static const struct msi_parent_ops virt_amdvi_msi_parent_ops = { - .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | - MSI_FLAG_MULTI_PCI_MSI, - .prefix = "vIR-", - .init_dev_msi_info = msi_parent_init_dev_msi_info, -}; - int amd_iommu_create_irq_domain(struct amd_iommu *iommu) { struct fwnode_handle *fn; @@ -3739,11 +3806,7 @@ int amd_iommu_create_irq_domain(struct amd_iommu *iommu) irq_domain_update_bus_token(iommu->ir_domain, DOMAIN_BUS_AMDVI); iommu->ir_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT | IRQ_DOMAIN_FLAG_ISOLATED_MSI; - - if (amd_iommu_np_cache) - iommu->ir_domain->msi_parent_ops = &virt_amdvi_msi_parent_ops; - else - iommu->ir_domain->msi_parent_ops = &amdvi_msi_parent_ops; + iommu->ir_domain->msi_parent_ops = &amdvi_msi_parent_ops; return 0; } diff --git a/drivers/iommu/amd/pasid.c b/drivers/iommu/amd/pasid.c new file mode 100644 index 000000000000..a68215f2b3e1 --- /dev/null +++ b/drivers/iommu/amd/pasid.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Advanced Micro Devices, Inc. + */ + +#define pr_fmt(fmt) "AMD-Vi: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) + +#include <linux/iommu.h> +#include <linux/mm_types.h> + +#include "amd_iommu.h" + +static inline bool is_pasid_enabled(struct iommu_dev_data *dev_data) +{ + if (dev_data->pasid_enabled && dev_data->max_pasids && + dev_data->gcr3_info.gcr3_tbl != NULL) + return true; + + return false; +} + +static inline bool is_pasid_valid(struct iommu_dev_data *dev_data, + ioasid_t pasid) +{ + if (pasid > 0 && pasid < dev_data->max_pasids) + return true; + + return false; +} + +static void remove_dev_pasid(struct pdom_dev_data *pdom_dev_data) +{ + /* Update GCR3 table and flush IOTLB */ + amd_iommu_clear_gcr3(pdom_dev_data->dev_data, pdom_dev_data->pasid); + + list_del(&pdom_dev_data->list); + kfree(pdom_dev_data); +} + +/* Clear PASID from device GCR3 table and remove pdom_dev_data from list */ +static void remove_pdom_dev_pasid(struct protection_domain *pdom, + struct device *dev, ioasid_t pasid) +{ + struct pdom_dev_data *pdom_dev_data; + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + + lockdep_assert_held(&pdom->lock); + + for_each_pdom_dev_data(pdom_dev_data, pdom) { + if (pdom_dev_data->dev_data == dev_data && + pdom_dev_data->pasid == pasid) { + remove_dev_pasid(pdom_dev_data); + break; + } + } +} + +static void sva_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + struct pdom_dev_data *pdom_dev_data; + struct protection_domain *sva_pdom; + unsigned long flags; + + sva_pdom = container_of(mn, struct protection_domain, mn); + + spin_lock_irqsave(&sva_pdom->lock, flags); + + for_each_pdom_dev_data(pdom_dev_data, sva_pdom) { + amd_iommu_dev_flush_pasid_pages(pdom_dev_data->dev_data, + pdom_dev_data->pasid, + start, end - start); + } + + spin_unlock_irqrestore(&sva_pdom->lock, flags); +} + +static void sva_mn_release(struct mmu_notifier *mn, struct mm_struct *mm) +{ + struct pdom_dev_data *pdom_dev_data, *next; + struct protection_domain *sva_pdom; + unsigned long flags; + + sva_pdom = container_of(mn, struct protection_domain, mn); + + spin_lock_irqsave(&sva_pdom->lock, flags); + + /* Assume dev_data_list contains same PASID with different devices */ + for_each_pdom_dev_data_safe(pdom_dev_data, next, sva_pdom) + remove_dev_pasid(pdom_dev_data); + + spin_unlock_irqrestore(&sva_pdom->lock, flags); +} + +static const struct mmu_notifier_ops sva_mn = { + .arch_invalidate_secondary_tlbs = sva_arch_invalidate_secondary_tlbs, + .release = sva_mn_release, +}; + +int iommu_sva_set_dev_pasid(struct iommu_domain *domain, + struct device *dev, ioasid_t pasid) +{ + struct pdom_dev_data *pdom_dev_data; + struct protection_domain *sva_pdom = to_pdomain(domain); + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + unsigned long flags; + int ret = -EINVAL; + + /* PASID zero is used for requests from the I/O device without PASID */ + if (!is_pasid_valid(dev_data, pasid)) + return ret; + + /* Make sure PASID is enabled */ + if (!is_pasid_enabled(dev_data)) + return ret; + + /* Add PASID to protection domain pasid list */ + pdom_dev_data = kzalloc(sizeof(*pdom_dev_data), GFP_KERNEL); + if (pdom_dev_data == NULL) + return ret; + + pdom_dev_data->pasid = pasid; + pdom_dev_data->dev_data = dev_data; + + spin_lock_irqsave(&sva_pdom->lock, flags); + + /* Setup GCR3 table */ + ret = amd_iommu_set_gcr3(dev_data, pasid, + iommu_virt_to_phys(domain->mm->pgd)); + if (ret) { + kfree(pdom_dev_data); + goto out_unlock; + } + + list_add(&pdom_dev_data->list, &sva_pdom->dev_data_list); + +out_unlock: + spin_unlock_irqrestore(&sva_pdom->lock, flags); + return ret; +} + +void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid, + struct iommu_domain *domain) +{ + struct protection_domain *sva_pdom; + unsigned long flags; + + if (!is_pasid_valid(dev_iommu_priv_get(dev), pasid)) + return; + + sva_pdom = to_pdomain(domain); + + spin_lock_irqsave(&sva_pdom->lock, flags); + + /* Remove PASID from dev_data_list */ + remove_pdom_dev_pasid(sva_pdom, dev, pasid); + + spin_unlock_irqrestore(&sva_pdom->lock, flags); +} + +static void iommu_sva_domain_free(struct iommu_domain *domain) +{ + struct protection_domain *sva_pdom = to_pdomain(domain); + + if (sva_pdom->mn.ops) + mmu_notifier_unregister(&sva_pdom->mn, domain->mm); + + amd_iommu_domain_free(domain); +} + +static const struct iommu_domain_ops amd_sva_domain_ops = { + .set_dev_pasid = iommu_sva_set_dev_pasid, + .free = iommu_sva_domain_free +}; + +struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev, + struct mm_struct *mm) +{ + struct protection_domain *pdom; + int ret; + + pdom = protection_domain_alloc(IOMMU_DOMAIN_SVA); + if (!pdom) + return ERR_PTR(-ENOMEM); + + pdom->domain.ops = &amd_sva_domain_ops; + pdom->mn.ops = &sva_mn; + + ret = mmu_notifier_register(&pdom->mn, mm); + if (ret) { + protection_domain_free(pdom); + return ERR_PTR(ret); + } + + return &pdom->domain; +} diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c new file mode 100644 index 000000000000..091423bb8aac --- /dev/null +++ b/drivers/iommu/amd/ppr.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023 Advanced Micro Devices, Inc. + */ + +#define pr_fmt(fmt) "AMD-Vi: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) + +#include <linux/amd-iommu.h> +#include <linux/delay.h> +#include <linux/mmu_notifier.h> + +#include <asm/iommu.h> + +#include "amd_iommu.h" +#include "amd_iommu_types.h" + +#include "../iommu-pages.h" + +int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu) +{ + iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, + PPR_LOG_SIZE); + return iommu->ppr_log ? 0 : -ENOMEM; +} + +void amd_iommu_enable_ppr_log(struct amd_iommu *iommu) +{ + u64 entry; + + if (iommu->ppr_log == NULL) + return; + + iommu_feature_enable(iommu, CONTROL_PPR_EN); + + entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512; + + memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET, + &entry, sizeof(entry)); + + /* set head and tail to zero manually */ + writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); + writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); + + iommu_feature_enable(iommu, CONTROL_PPRINT_EN); + iommu_feature_enable(iommu, CONTROL_PPRLOG_EN); +} + +void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu) +{ + iommu_free_pages(iommu->ppr_log, get_order(PPR_LOG_SIZE)); +} + +/* + * This function restarts ppr logging in case the IOMMU experienced + * PPR log overflow. + */ +void amd_iommu_restart_ppr_log(struct amd_iommu *iommu) +{ + amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN, + CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK, + MMIO_STATUS_PPR_OVERFLOW_MASK); +} + +static inline u32 ppr_flag_to_fault_perm(u16 flag) +{ + int perm = 0; + + if (flag & PPR_FLAG_READ) + perm |= IOMMU_FAULT_PERM_READ; + if (flag & PPR_FLAG_WRITE) + perm |= IOMMU_FAULT_PERM_WRITE; + if (flag & PPR_FLAG_EXEC) + perm |= IOMMU_FAULT_PERM_EXEC; + if (!(flag & PPR_FLAG_US)) + perm |= IOMMU_FAULT_PERM_PRIV; + + return perm; +} + +static bool ppr_is_valid(struct amd_iommu *iommu, u64 *raw) +{ + struct device *dev = iommu->iommu.dev; + u16 devid = PPR_DEVID(raw[0]); + + if (!(PPR_FLAGS(raw[0]) & PPR_FLAG_GN)) { + dev_dbg(dev, "PPR logged [Request ignored due to GN=0 (device=%04x:%02x:%02x.%x " + "pasid=0x%05llx address=0x%llx flags=0x%04llx tag=0x%03llx]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + PPR_PASID(raw[0]), raw[1], PPR_FLAGS(raw[0]), PPR_TAG(raw[0])); + return false; + } + + if (PPR_FLAGS(raw[0]) & PPR_FLAG_RVSD) { + dev_dbg(dev, "PPR logged [Invalid request format (device=%04x:%02x:%02x.%x " + "pasid=0x%05llx address=0x%llx flags=0x%04llx tag=0x%03llx]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + PPR_PASID(raw[0]), raw[1], PPR_FLAGS(raw[0]), PPR_TAG(raw[0])); + return false; + } + + return true; +} + +static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw) +{ + struct iommu_dev_data *dev_data; + struct iopf_fault event; + struct pci_dev *pdev; + u16 devid = PPR_DEVID(raw[0]); + + if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) { + pr_info_ratelimited("Unknown PPR request received\n"); + return; + } + + pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, + PCI_BUS_NUM(devid), devid & 0xff); + if (!pdev) + return; + + if (!ppr_is_valid(iommu, raw)) + goto out; + + memset(&event, 0, sizeof(struct iopf_fault)); + + event.fault.type = IOMMU_FAULT_PAGE_REQ; + event.fault.prm.perm = ppr_flag_to_fault_perm(PPR_FLAGS(raw[0])); + event.fault.prm.addr = (u64)(raw[1] & PAGE_MASK); + event.fault.prm.pasid = PPR_PASID(raw[0]); + event.fault.prm.grpid = PPR_TAG(raw[0]) & 0x1FF; + + /* + * PASID zero is used for requests from the I/O device without + * a PASID + */ + dev_data = dev_iommu_priv_get(&pdev->dev); + if (event.fault.prm.pasid == 0 || + event.fault.prm.pasid >= dev_data->max_pasids) { + pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n", + event.fault.prm.pasid, pdev->dev.id); + goto out; + } + + event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; + event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; + if (PPR_TAG(raw[0]) & 0x200) + event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; + + /* Submit event */ + iommu_report_device_fault(&pdev->dev, &event); + + return; + +out: + /* Nobody cared, abort */ + amd_iommu_complete_ppr(&pdev->dev, PPR_PASID(raw[0]), + IOMMU_PAGE_RESP_FAILURE, + PPR_TAG(raw[0]) & 0x1FF); +} + +void amd_iommu_poll_ppr_log(struct amd_iommu *iommu) +{ + u32 head, tail; + + if (iommu->ppr_log == NULL) + return; + + head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); + tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); + + while (head != tail) { + volatile u64 *raw; + u64 entry[2]; + int i; + + raw = (u64 *)(iommu->ppr_log + head); + + /* + * Hardware bug: Interrupt may arrive before the entry is + * written to memory. If this happens we need to wait for the + * entry to arrive. + */ + for (i = 0; i < LOOP_TIMEOUT; ++i) { + if (PPR_REQ_TYPE(raw[0]) != 0) + break; + udelay(1); + } + + /* Avoid memcpy function-call overhead */ + entry[0] = raw[0]; + entry[1] = raw[1]; + + /* + * To detect the hardware errata 733 we need to clear the + * entry back to zero. This issue does not exist on SNP + * enabled system. Also this buffer is not writeable on + * SNP enabled system. + */ + if (!amd_iommu_snp_en) + raw[0] = raw[1] = 0UL; + + /* Update head pointer of hardware ring-buffer */ + head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE; + writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); + + /* Handle PPR entry */ + iommu_call_iopf_notifier(iommu, entry); + } +} + +/************************************************************** + * + * IOPF handling stuff + */ + +/* Setup per-IOMMU IOPF queue if not exist. */ +int amd_iommu_iopf_init(struct amd_iommu *iommu) +{ + int ret = 0; + + if (iommu->iopf_queue) + return ret; + + snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), + "amdiommu-%#x-iopfq", + PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, iommu->devid)); + + iommu->iopf_queue = iopf_queue_alloc(iommu->iopfq_name); + if (!iommu->iopf_queue) + ret = -ENOMEM; + + return ret; +} + +/* Destroy per-IOMMU IOPF queue if no longer needed. */ +void amd_iommu_iopf_uninit(struct amd_iommu *iommu) +{ + iopf_queue_free(iommu->iopf_queue); + iommu->iopf_queue = NULL; +} + +void amd_iommu_page_response(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *resp) +{ + amd_iommu_complete_ppr(dev, resp->pasid, resp->code, resp->grpid); +} + +int amd_iommu_iopf_add_device(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data) +{ + unsigned long flags; + int ret = 0; + + if (!dev_data->pri_enabled) + return ret; + + raw_spin_lock_irqsave(&iommu->lock, flags); + + if (!iommu->iopf_queue) { + ret = -EINVAL; + goto out_unlock; + } + + ret = iopf_queue_add_device(iommu->iopf_queue, dev_data->dev); + if (ret) + goto out_unlock; + + dev_data->ppr = true; + +out_unlock: + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +/* Its assumed that caller has verified that device was added to iopf queue */ +void amd_iommu_iopf_remove_device(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&iommu->lock, flags); + + iopf_queue_remove_device(iommu->iopf_queue, dev_data->dev); + dev_data->ppr = false; + + raw_spin_unlock_irqrestore(&iommu->lock, flags); +} diff --git a/drivers/iommu/arm/arm-smmu-v3/Makefile b/drivers/iommu/arm/arm-smmu-v3/Makefile index 54feb1ecccad..014a997753a8 100644 --- a/drivers/iommu/arm/arm-smmu-v3/Makefile +++ b/drivers/iommu/arm/arm-smmu-v3/Makefile @@ -3,3 +3,5 @@ obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o arm_smmu_v3-objs-y += arm-smmu-v3.o arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o arm_smmu_v3-objs := $(arm_smmu_v3-objs-y) + +obj-$(CONFIG_ARM_SMMU_V3_KUNIT_TEST) += arm-smmu-v3-test.o diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 2cd433a9c8a0..e490ffb38015 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -8,6 +8,7 @@ #include <linux/mmu_notifier.h> #include <linux/sched/mm.h> #include <linux/slab.h> +#include <kunit/visibility.h> #include "arm-smmu-v3.h" #include "../../io-pgtable-arm.h" @@ -34,21 +35,25 @@ struct arm_smmu_bond { static DEFINE_MUTEX(sva_lock); -/* - * Write the CD to the CD tables for all masters that this domain is attached - * to. Note that this is only used to update existing CD entries in the target - * CD table, for which it's assumed that arm_smmu_write_ctx_desc can't fail. - */ -static void arm_smmu_update_ctx_desc_devices(struct arm_smmu_domain *smmu_domain, - int ssid, - struct arm_smmu_ctx_desc *cd) +static void +arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) { struct arm_smmu_master *master; + struct arm_smmu_cd target_cd; unsigned long flags; spin_lock_irqsave(&smmu_domain->devices_lock, flags); list_for_each_entry(master, &smmu_domain->devices, domain_head) { - arm_smmu_write_ctx_desc(master, ssid, cd); + struct arm_smmu_cd *cdptr; + + /* S1 domains only support RID attachment right now */ + cdptr = arm_smmu_get_cd_ptr(master, IOMMU_NO_PASID); + if (WARN_ON(!cdptr)) + continue; + + arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); + arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr, + &target_cd); } spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); } @@ -96,7 +101,7 @@ arm_smmu_share_asid(struct mm_struct *mm, u16 asid) * be some overlap between use of both ASIDs, until we invalidate the * TLB. */ - arm_smmu_update_ctx_desc_devices(smmu_domain, IOMMU_NO_PASID, cd); + arm_smmu_update_s1_domain_cd_entry(smmu_domain); /* Invalidate TLB entries previously associated with that context */ arm_smmu_tlb_inv_asid(smmu, asid); @@ -105,11 +110,87 @@ arm_smmu_share_asid(struct mm_struct *mm, u16 asid) return NULL; } +static u64 page_size_to_cd(void) +{ + static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K || + PAGE_SIZE == SZ_64K); + if (PAGE_SIZE == SZ_64K) + return ARM_LPAE_TCR_TG0_64K; + if (PAGE_SIZE == SZ_16K) + return ARM_LPAE_TCR_TG0_16K; + return ARM_LPAE_TCR_TG0_4K; +} + +VISIBLE_IF_KUNIT +void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, + struct arm_smmu_master *master, struct mm_struct *mm, + u16 asid) +{ + u64 par; + + memset(target, 0, sizeof(*target)); + + par = cpuid_feature_extract_unsigned_field( + read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1), + ID_AA64MMFR0_EL1_PARANGE_SHIFT); + + target->data[0] = cpu_to_le64( + CTXDESC_CD_0_TCR_EPD1 | +#ifdef __BIG_ENDIAN + CTXDESC_CD_0_ENDI | +#endif + CTXDESC_CD_0_V | + FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par) | + CTXDESC_CD_0_AA64 | + (master->stall_enabled ? CTXDESC_CD_0_S : 0) | + CTXDESC_CD_0_R | + CTXDESC_CD_0_A | + CTXDESC_CD_0_ASET | + FIELD_PREP(CTXDESC_CD_0_ASID, asid)); + + /* + * If no MM is passed then this creates a SVA entry that faults + * everything. arm_smmu_write_cd_entry() can hitlessly go between these + * two entries types since TTB0 is ignored by HW when EPD0 is set. + */ + if (mm) { + target->data[0] |= cpu_to_le64( + FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, + 64ULL - vabits_actual) | + FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) | + FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, + ARM_LPAE_TCR_RGN_WBWA) | + FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, + ARM_LPAE_TCR_RGN_WBWA) | + FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS)); + + target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) & + CTXDESC_CD_1_TTB0_MASK); + } else { + target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0); + + /* + * Disable stall and immediately generate an abort if stall + * disable is permitted. This speeds up cleanup for an unclean + * exit if the device is still doing a lot of DMA. + */ + if (!(master->smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) + target->data[0] &= + cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R)); + } + + /* + * MAIR value is pretty much constant and global, so we can just get it + * from the current CPU register + */ + target->data[3] = cpu_to_le64(read_sysreg(mair_el1)); +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd); + static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) { u16 asid; int err = 0; - u64 tcr, par, reg; struct arm_smmu_ctx_desc *cd; struct arm_smmu_ctx_desc *ret = NULL; @@ -143,39 +224,6 @@ static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) if (err) goto out_free_asid; - tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) | - FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) | - FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, ARM_LPAE_TCR_RGN_WBWA) | - FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS) | - CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; - - switch (PAGE_SIZE) { - case SZ_4K: - tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_4K); - break; - case SZ_16K: - tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_16K); - break; - case SZ_64K: - tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_64K); - break; - default: - WARN_ON(1); - err = -EINVAL; - goto out_free_asid; - } - - reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); - par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); - tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par); - - cd->ttbr = virt_to_phys(mm->pgd); - cd->tcr = tcr; - /* - * MAIR value is pretty much constant and global, so we can just get it - * from the current CPU register - */ - cd->mair = read_sysreg(mair_el1); cd->asid = asid; cd->mm = mm; @@ -253,6 +301,8 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) { struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); struct arm_smmu_domain *smmu_domain = smmu_mn->domain; + struct arm_smmu_master *master; + unsigned long flags; mutex_lock(&sva_lock); if (smmu_mn->cleared) { @@ -264,8 +314,19 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events, * but disable translation. */ - arm_smmu_update_ctx_desc_devices(smmu_domain, mm_get_enqcmd_pasid(mm), - &quiet_cd); + spin_lock_irqsave(&smmu_domain->devices_lock, flags); + list_for_each_entry(master, &smmu_domain->devices, domain_head) { + struct arm_smmu_cd target; + struct arm_smmu_cd *cdptr; + + cdptr = arm_smmu_get_cd_ptr(master, mm_get_enqcmd_pasid(mm)); + if (WARN_ON(!cdptr)) + continue; + arm_smmu_make_sva_cd(&target, master, NULL, smmu_mn->cd->asid); + arm_smmu_write_cd_entry(master, mm_get_enqcmd_pasid(mm), cdptr, + &target); + } + spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid); arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), 0, 0); @@ -360,6 +421,8 @@ static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid, struct mm_struct *mm) { int ret; + struct arm_smmu_cd target; + struct arm_smmu_cd *cdptr; struct arm_smmu_bond *bond; struct arm_smmu_master *master = dev_iommu_priv_get(dev); struct iommu_domain *domain = iommu_get_domain_for_dev(dev); @@ -386,9 +449,13 @@ static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid, goto err_free_bond; } - ret = arm_smmu_write_ctx_desc(master, pasid, bond->smmu_mn->cd); - if (ret) + cdptr = arm_smmu_alloc_cd_ptr(master, mm_get_enqcmd_pasid(mm)); + if (!cdptr) { + ret = -ENOMEM; goto err_put_notifier; + } + arm_smmu_make_sva_cd(&target, master, mm, bond->smmu_mn->cd->asid); + arm_smmu_write_cd_entry(master, pasid, cdptr, &target); list_add(&bond->list, &master->bonds); return 0; @@ -546,7 +613,7 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, mutex_lock(&sva_lock); - arm_smmu_write_ctx_desc(master, id, NULL); + arm_smmu_clear_cd(master, id); list_for_each_entry(t, &master->bonds, list) { if (t->mm == mm) { @@ -569,6 +636,9 @@ static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, int ret = 0; struct mm_struct *mm = domain->mm; + if (mm_get_enqcmd_pasid(mm) != id) + return -EINVAL; + mutex_lock(&sva_lock); ret = __arm_smmu_sva_bind(dev, id, mm); mutex_unlock(&sva_lock); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c new file mode 100644 index 000000000000..315e487fd990 --- /dev/null +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -0,0 +1,468 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2024 Google LLC. + */ +#include <kunit/test.h> +#include <linux/io-pgtable.h> + +#include "arm-smmu-v3.h" + +struct arm_smmu_test_writer { + struct arm_smmu_entry_writer writer; + struct kunit *test; + const __le64 *init_entry; + const __le64 *target_entry; + __le64 *entry; + + bool invalid_entry_written; + unsigned int num_syncs; +}; + +#define NUM_ENTRY_QWORDS 8 +#define NUM_EXPECTED_SYNCS(x) x + +static struct arm_smmu_ste bypass_ste; +static struct arm_smmu_ste abort_ste; +static struct arm_smmu_device smmu = { + .features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR +}; +static struct mm_struct sva_mm = { + .pgd = (void *)0xdaedbeefdeadbeefULL, +}; + +static bool arm_smmu_entry_differs_in_used_bits(const __le64 *entry, + const __le64 *used_bits, + const __le64 *target, + unsigned int length) +{ + bool differs = false; + unsigned int i; + + for (i = 0; i < length; i++) { + if ((entry[i] & used_bits[i]) != target[i]) + differs = true; + } + return differs; +} + +static void +arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) +{ + struct arm_smmu_test_writer *test_writer = + container_of(writer, struct arm_smmu_test_writer, writer); + __le64 *entry_used_bits; + + entry_used_bits = kunit_kzalloc( + test_writer->test, sizeof(*entry_used_bits) * NUM_ENTRY_QWORDS, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test_writer->test, entry_used_bits); + + pr_debug("STE value is now set to: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, + test_writer->entry, + NUM_ENTRY_QWORDS * sizeof(*test_writer->entry), + false); + + test_writer->num_syncs += 1; + if (!test_writer->entry[0]) { + test_writer->invalid_entry_written = true; + } else { + /* + * At any stage in a hitless transition, the entry must be + * equivalent to either the initial entry or the target entry + * when only considering the bits used by the current + * configuration. + */ + writer->ops->get_used(test_writer->entry, entry_used_bits); + KUNIT_EXPECT_FALSE( + test_writer->test, + arm_smmu_entry_differs_in_used_bits( + test_writer->entry, entry_used_bits, + test_writer->init_entry, NUM_ENTRY_QWORDS) && + arm_smmu_entry_differs_in_used_bits( + test_writer->entry, entry_used_bits, + test_writer->target_entry, + NUM_ENTRY_QWORDS)); + } +} + +static void +arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, + const __le64 *ste) +{ + __le64 used_bits[NUM_ENTRY_QWORDS] = {}; + + arm_smmu_get_ste_used(ste, used_bits); + pr_debug("STE used bits: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, used_bits, + sizeof(used_bits), false); +} + +static const struct arm_smmu_entry_writer_ops test_ste_ops = { + .sync = arm_smmu_test_writer_record_syncs, + .get_used = arm_smmu_get_ste_used, +}; + +static const struct arm_smmu_entry_writer_ops test_cd_ops = { + .sync = arm_smmu_test_writer_record_syncs, + .get_used = arm_smmu_get_cd_used, +}; + +static void arm_smmu_v3_test_ste_expect_transition( + struct kunit *test, const struct arm_smmu_ste *cur, + const struct arm_smmu_ste *target, unsigned int num_syncs_expected, + bool hitless) +{ + struct arm_smmu_ste cur_copy = *cur; + struct arm_smmu_test_writer test_writer = { + .writer = { + .ops = &test_ste_ops, + }, + .test = test, + .init_entry = cur->data, + .target_entry = target->data, + .entry = cur_copy.data, + .num_syncs = 0, + .invalid_entry_written = false, + + }; + + pr_debug("STE initial value: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, cur_copy.data, + sizeof(cur_copy), false); + arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, cur->data); + pr_debug("STE target value: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, target->data, + sizeof(cur_copy), false); + arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, + target->data); + + arm_smmu_write_entry(&test_writer.writer, cur_copy.data, target->data); + + KUNIT_EXPECT_EQ(test, test_writer.invalid_entry_written, !hitless); + KUNIT_EXPECT_EQ(test, test_writer.num_syncs, num_syncs_expected); + KUNIT_EXPECT_MEMEQ(test, target->data, cur_copy.data, sizeof(cur_copy)); +} + +static void arm_smmu_v3_test_ste_expect_hitless_transition( + struct kunit *test, const struct arm_smmu_ste *cur, + const struct arm_smmu_ste *target, unsigned int num_syncs_expected) +{ + arm_smmu_v3_test_ste_expect_transition(test, cur, target, + num_syncs_expected, true); +} + +static const dma_addr_t fake_cdtab_dma_addr = 0xF0F0F0F0F0F0; + +static void arm_smmu_test_make_cdtable_ste(struct arm_smmu_ste *ste, + const dma_addr_t dma_addr) +{ + struct arm_smmu_master master = { + .cd_table.cdtab_dma = dma_addr, + .cd_table.s1cdmax = 0xFF, + .cd_table.s1fmt = STRTAB_STE_0_S1FMT_64K_L2, + .smmu = &smmu, + }; + + arm_smmu_make_cdtable_ste(ste, &master); +} + +static void arm_smmu_v3_write_ste_test_bypass_to_abort(struct kunit *test) +{ + /* + * Bypass STEs has used bits in the first two Qwords, while abort STEs + * only have used bits in the first QWord. Transitioning from bypass to + * abort requires two syncs: the first to set the first qword and make + * the STE into an abort, the second to clean up the second qword. + */ + arm_smmu_v3_test_ste_expect_hitless_transition( + test, &bypass_ste, &abort_ste, NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_abort_to_bypass(struct kunit *test) +{ + /* + * Transitioning from abort to bypass also requires two syncs: the first + * to set the second qword data required by the bypass STE, and the + * second to set the first qword and switch to bypass. + */ + arm_smmu_v3_test_ste_expect_hitless_transition( + test, &abort_ste, &bypass_ste, NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_cdtable_to_abort(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &abort_ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_abort_to_cdtable(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &abort_ste, &ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_cdtable_to_bypass(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &bypass_ste, + NUM_EXPECTED_SYNCS(3)); +} + +static void arm_smmu_v3_write_ste_test_bypass_to_cdtable(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &bypass_ste, &ste, + NUM_EXPECTED_SYNCS(3)); +} + +static void arm_smmu_test_make_s2_ste(struct arm_smmu_ste *ste, + bool ats_enabled) +{ + struct arm_smmu_master master = { + .smmu = &smmu, + .ats_enabled = ats_enabled, + }; + struct io_pgtable io_pgtable = {}; + struct arm_smmu_domain smmu_domain = { + .pgtbl_ops = &io_pgtable.ops, + }; + + io_pgtable.cfg.arm_lpae_s2_cfg.vttbr = 0xdaedbeefdeadbeefULL; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.ps = 1; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.tg = 2; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.sh = 3; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.orgn = 1; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.irgn = 2; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.sl = 3; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.tsz = 4; + + arm_smmu_make_s2_domain_ste(ste, &master, &smmu_domain); +} + +static void arm_smmu_v3_write_ste_test_s2_to_abort(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_s2_ste(&ste, true); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &abort_ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_abort_to_s2(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_s2_ste(&ste, true); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &abort_ste, &ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_s2_to_bypass(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_s2_ste(&ste, true); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &bypass_ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_bypass_to_s2(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_s2_ste(&ste, true); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &bypass_ste, &ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_test_cd_expect_transition( + struct kunit *test, const struct arm_smmu_cd *cur, + const struct arm_smmu_cd *target, unsigned int num_syncs_expected, + bool hitless) +{ + struct arm_smmu_cd cur_copy = *cur; + struct arm_smmu_test_writer test_writer = { + .writer = { + .ops = &test_cd_ops, + }, + .test = test, + .init_entry = cur->data, + .target_entry = target->data, + .entry = cur_copy.data, + .num_syncs = 0, + .invalid_entry_written = false, + + }; + + pr_debug("CD initial value: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, cur_copy.data, + sizeof(cur_copy), false); + arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, cur->data); + pr_debug("CD target value: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, target->data, + sizeof(cur_copy), false); + arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, + target->data); + + arm_smmu_write_entry(&test_writer.writer, cur_copy.data, target->data); + + KUNIT_EXPECT_EQ(test, test_writer.invalid_entry_written, !hitless); + KUNIT_EXPECT_EQ(test, test_writer.num_syncs, num_syncs_expected); + KUNIT_EXPECT_MEMEQ(test, target->data, cur_copy.data, sizeof(cur_copy)); +} + +static void arm_smmu_v3_test_cd_expect_non_hitless_transition( + struct kunit *test, const struct arm_smmu_cd *cur, + const struct arm_smmu_cd *target, unsigned int num_syncs_expected) +{ + arm_smmu_v3_test_cd_expect_transition(test, cur, target, + num_syncs_expected, false); +} + +static void arm_smmu_v3_test_cd_expect_hitless_transition( + struct kunit *test, const struct arm_smmu_cd *cur, + const struct arm_smmu_cd *target, unsigned int num_syncs_expected) +{ + arm_smmu_v3_test_cd_expect_transition(test, cur, target, + num_syncs_expected, true); +} + +static void arm_smmu_test_make_s1_cd(struct arm_smmu_cd *cd, unsigned int asid) +{ + struct arm_smmu_master master = { + .smmu = &smmu, + }; + struct io_pgtable io_pgtable = {}; + struct arm_smmu_domain smmu_domain = { + .pgtbl_ops = &io_pgtable.ops, + .cd = { + .asid = asid, + }, + }; + + io_pgtable.cfg.arm_lpae_s1_cfg.ttbr = 0xdaedbeefdeadbeefULL; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.ips = 1; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.tg = 2; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.sh = 3; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.orgn = 1; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.irgn = 2; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.tsz = 4; + io_pgtable.cfg.arm_lpae_s1_cfg.mair = 0xabcdef012345678ULL; + + arm_smmu_make_s1_cd(cd, &master, &smmu_domain); +} + +static void arm_smmu_v3_write_cd_test_s1_clear(struct kunit *test) +{ + struct arm_smmu_cd cd = {}; + struct arm_smmu_cd cd_2; + + arm_smmu_test_make_s1_cd(&cd_2, 1997); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_cd_test_s1_change_asid(struct kunit *test) +{ + struct arm_smmu_cd cd = {}; + struct arm_smmu_cd cd_2; + + arm_smmu_test_make_s1_cd(&cd, 778); + arm_smmu_test_make_s1_cd(&cd_2, 1997); + arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd, &cd_2, + NUM_EXPECTED_SYNCS(1)); + arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd_2, &cd, + NUM_EXPECTED_SYNCS(1)); +} + +static void arm_smmu_test_make_sva_cd(struct arm_smmu_cd *cd, unsigned int asid) +{ + struct arm_smmu_master master = { + .smmu = &smmu, + }; + + arm_smmu_make_sva_cd(cd, &master, &sva_mm, asid); +} + +static void arm_smmu_test_make_sva_release_cd(struct arm_smmu_cd *cd, + unsigned int asid) +{ + struct arm_smmu_master master = { + .smmu = &smmu, + }; + + arm_smmu_make_sva_cd(cd, &master, NULL, asid); +} + +static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test) +{ + struct arm_smmu_cd cd = {}; + struct arm_smmu_cd cd_2; + + arm_smmu_test_make_sva_cd(&cd_2, 1997); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test) +{ + struct arm_smmu_cd cd; + struct arm_smmu_cd cd_2; + + arm_smmu_test_make_sva_cd(&cd, 1997); + arm_smmu_test_make_sva_release_cd(&cd_2, 1997); + arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd, &cd_2, + NUM_EXPECTED_SYNCS(2)); + arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd_2, &cd, + NUM_EXPECTED_SYNCS(2)); +} + +static struct kunit_case arm_smmu_v3_test_cases[] = { + KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_abort), + KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_bypass), + KUNIT_CASE(arm_smmu_v3_write_ste_test_cdtable_to_abort), + KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_cdtable), + KUNIT_CASE(arm_smmu_v3_write_ste_test_cdtable_to_bypass), + KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_cdtable), + KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_abort), + KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_s2), + KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_bypass), + KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_s2), + KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_clear), + KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_change_asid), + KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear), + KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release), + {}, +}; + +static int arm_smmu_v3_test_suite_init(struct kunit_suite *test) +{ + arm_smmu_make_bypass_ste(&smmu, &bypass_ste); + arm_smmu_make_abort_ste(&abort_ste); + return 0; +} + +static struct kunit_suite arm_smmu_v3_test_module = { + .name = "arm-smmu-v3-kunit-test", + .suite_init = arm_smmu_v3_test_suite_init, + .test_cases = arm_smmu_v3_test_cases, +}; +kunit_test_suites(&arm_smmu_v3_test_module); + +MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 41f93c3ab160..ab415e107054 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -26,15 +26,11 @@ #include <linux/pci.h> #include <linux/pci-ats.h> #include <linux/platform_device.h> +#include <kunit/visibility.h> #include "arm-smmu-v3.h" #include "../../dma-iommu.h" -static bool disable_bypass = true; -module_param(disable_bypass, bool, 0444); -MODULE_PARM_DESC(disable_bypass, - "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); - static bool disable_msipolling; module_param(disable_msipolling, bool, 0444); MODULE_PARM_DESC(disable_msipolling, @@ -47,8 +43,9 @@ enum arm_smmu_msi_index { ARM_SMMU_MAX_MSIS, }; -static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, - ioasid_t sid); +#define NUM_ENTRY_QWORDS 8 +static_assert(sizeof(struct arm_smmu_ste) == NUM_ENTRY_QWORDS * sizeof(u64)); +static_assert(sizeof(struct arm_smmu_cd) == NUM_ENTRY_QWORDS * sizeof(u64)); static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = { [EVTQ_MSI_INDEX] = { @@ -76,12 +73,6 @@ struct arm_smmu_option_prop { DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa); DEFINE_MUTEX(arm_smmu_asid_lock); -/* - * Special value used by SVA when a process dies, to quiesce a CD without - * disabling it. - */ -struct arm_smmu_ctx_desc quiet_cd = { 0 }; - static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" }, { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"}, @@ -90,6 +81,7 @@ static struct arm_smmu_option_prop arm_smmu_options[] = { static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, struct arm_smmu_device *smmu); +static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master); static void parse_driver_options(struct arm_smmu_device *smmu) { @@ -977,44 +969,45 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) * would be nice if this was complete according to the spec, but minimally it * has to capture the bits this driver uses. */ -static void arm_smmu_get_ste_used(const struct arm_smmu_ste *ent, - struct arm_smmu_ste *used_bits) +VISIBLE_IF_KUNIT +void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) { - unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent->data[0])); + unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent[0])); - used_bits->data[0] = cpu_to_le64(STRTAB_STE_0_V); - if (!(ent->data[0] & cpu_to_le64(STRTAB_STE_0_V))) + used_bits[0] = cpu_to_le64(STRTAB_STE_0_V); + if (!(ent[0] & cpu_to_le64(STRTAB_STE_0_V))) return; - used_bits->data[0] |= cpu_to_le64(STRTAB_STE_0_CFG); + used_bits[0] |= cpu_to_le64(STRTAB_STE_0_CFG); /* S1 translates */ if (cfg & BIT(0)) { - used_bits->data[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT | - STRTAB_STE_0_S1CTXPTR_MASK | - STRTAB_STE_0_S1CDMAX); - used_bits->data[1] |= + used_bits[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT | + STRTAB_STE_0_S1CTXPTR_MASK | + STRTAB_STE_0_S1CDMAX); + used_bits[1] |= cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR | STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH | STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW | STRTAB_STE_1_EATS); - used_bits->data[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID); + used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID); } /* S2 translates */ if (cfg & BIT(1)) { - used_bits->data[1] |= + used_bits[1] |= cpu_to_le64(STRTAB_STE_1_EATS | STRTAB_STE_1_SHCFG); - used_bits->data[2] |= + used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR | STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI | STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2R); - used_bits->data[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK); + used_bits[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK); } if (cfg == STRTAB_STE_0_CFG_BYPASS) - used_bits->data[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG); + used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG); } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used); /* * Figure out if we can do a hitless update of entry to become target. Returns a @@ -1022,57 +1015,55 @@ static void arm_smmu_get_ste_used(const struct arm_smmu_ste *ent, * unused_update is an intermediate value of entry that has unused bits set to * their new values. */ -static u8 arm_smmu_entry_qword_diff(const struct arm_smmu_ste *entry, - const struct arm_smmu_ste *target, - struct arm_smmu_ste *unused_update) +static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, + const __le64 *entry, const __le64 *target, + __le64 *unused_update) { - struct arm_smmu_ste target_used = {}; - struct arm_smmu_ste cur_used = {}; + __le64 target_used[NUM_ENTRY_QWORDS] = {}; + __le64 cur_used[NUM_ENTRY_QWORDS] = {}; u8 used_qword_diff = 0; unsigned int i; - arm_smmu_get_ste_used(entry, &cur_used); - arm_smmu_get_ste_used(target, &target_used); + writer->ops->get_used(entry, cur_used); + writer->ops->get_used(target, target_used); - for (i = 0; i != ARRAY_SIZE(target_used.data); i++) { + for (i = 0; i != NUM_ENTRY_QWORDS; i++) { /* * Check that masks are up to date, the make functions are not * allowed to set a bit to 1 if the used function doesn't say it * is used. */ - WARN_ON_ONCE(target->data[i] & ~target_used.data[i]); + WARN_ON_ONCE(target[i] & ~target_used[i]); /* Bits can change because they are not currently being used */ - unused_update->data[i] = (entry->data[i] & cur_used.data[i]) | - (target->data[i] & ~cur_used.data[i]); + unused_update[i] = (entry[i] & cur_used[i]) | + (target[i] & ~cur_used[i]); /* * Each bit indicates that a used bit in a qword needs to be * changed after unused_update is applied. */ - if ((unused_update->data[i] & target_used.data[i]) != - target->data[i]) + if ((unused_update[i] & target_used[i]) != target[i]) used_qword_diff |= 1 << i; } return used_qword_diff; } -static bool entry_set(struct arm_smmu_device *smmu, ioasid_t sid, - struct arm_smmu_ste *entry, - const struct arm_smmu_ste *target, unsigned int start, +static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, + const __le64 *target, unsigned int start, unsigned int len) { bool changed = false; unsigned int i; for (i = start; len != 0; len--, i++) { - if (entry->data[i] != target->data[i]) { - WRITE_ONCE(entry->data[i], target->data[i]); + if (entry[i] != target[i]) { + WRITE_ONCE(entry[i], target[i]); changed = true; } } if (changed) - arm_smmu_sync_ste_for_sid(smmu, sid); + writer->ops->sync(writer); return changed; } @@ -1102,24 +1093,22 @@ static bool entry_set(struct arm_smmu_device *smmu, ioasid_t sid, * V=0 process. This relies on the IGNORED behavior described in the * specification. */ -static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, - struct arm_smmu_ste *entry, - const struct arm_smmu_ste *target) +VISIBLE_IF_KUNIT +void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry, + const __le64 *target) { - unsigned int num_entry_qwords = ARRAY_SIZE(target->data); - struct arm_smmu_device *smmu = master->smmu; - struct arm_smmu_ste unused_update; + __le64 unused_update[NUM_ENTRY_QWORDS]; u8 used_qword_diff; used_qword_diff = - arm_smmu_entry_qword_diff(entry, target, &unused_update); + arm_smmu_entry_qword_diff(writer, entry, target, unused_update); if (hweight8(used_qword_diff) == 1) { /* * Only one qword needs its used bits to be changed. This is a - * hitless update, update all bits the current STE is ignoring - * to their new values, then update a single "critical qword" to - * change the STE and finally 0 out any bits that are now unused - * in the target configuration. + * hitless update, update all bits the current STE/CD is + * ignoring to their new values, then update a single "critical + * qword" to change the STE/CD and finally 0 out any bits that + * are now unused in the target configuration. */ unsigned int critical_qword_index = ffs(used_qword_diff) - 1; @@ -1128,22 +1117,21 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, * writing it in the next step anyways. This can save a sync * when the only change is in that qword. */ - unused_update.data[critical_qword_index] = - entry->data[critical_qword_index]; - entry_set(smmu, sid, entry, &unused_update, 0, num_entry_qwords); - entry_set(smmu, sid, entry, target, critical_qword_index, 1); - entry_set(smmu, sid, entry, target, 0, num_entry_qwords); + unused_update[critical_qword_index] = + entry[critical_qword_index]; + entry_set(writer, entry, unused_update, 0, NUM_ENTRY_QWORDS); + entry_set(writer, entry, target, critical_qword_index, 1); + entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS); } else if (used_qword_diff) { /* * At least two qwords need their inuse bits to be changed. This * requires a breaking update, zero the V bit, write all qwords * but 0, then set qword 0 */ - unused_update.data[0] = entry->data[0] & - cpu_to_le64(~STRTAB_STE_0_V); - entry_set(smmu, sid, entry, &unused_update, 0, 1); - entry_set(smmu, sid, entry, target, 1, num_entry_qwords - 1); - entry_set(smmu, sid, entry, target, 0, 1); + unused_update[0] = 0; + entry_set(writer, entry, unused_update, 0, 1); + entry_set(writer, entry, target, 1, NUM_ENTRY_QWORDS - 1); + entry_set(writer, entry, target, 0, 1); } else { /* * No inuse bit changed. Sanity check that all unused bits are 0 @@ -1151,20 +1139,10 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, * compute_qword_diff(). */ WARN_ON_ONCE( - entry_set(smmu, sid, entry, target, 0, num_entry_qwords)); - } - - /* It's likely that we'll want to use the new STE soon */ - if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) { - struct arm_smmu_cmdq_ent - prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG, - .prefetch = { - .sid = sid, - } }; - - arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); + entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS)); } } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry); static void arm_smmu_sync_cd(struct arm_smmu_master *master, int ssid, bool leaf) @@ -1210,117 +1188,166 @@ static void arm_smmu_write_cd_l1_desc(__le64 *dst, u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V; - /* See comment in arm_smmu_write_ctx_desc() */ + /* The HW has 64 bit atomicity with stores to the L2 CD table */ WRITE_ONCE(*dst, cpu_to_le64(val)); } -static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid) +struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, + u32 ssid) { - __le64 *l1ptr; - unsigned int idx; struct arm_smmu_l1_ctx_desc *l1_desc; - struct arm_smmu_device *smmu = master->smmu; struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; + if (!cd_table->cdtab) + return NULL; + if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR) - return cd_table->cdtab + ssid * CTXDESC_CD_DWORDS; + return (struct arm_smmu_cd *)(cd_table->cdtab + + ssid * CTXDESC_CD_DWORDS); - idx = ssid >> CTXDESC_SPLIT; - l1_desc = &cd_table->l1_desc[idx]; - if (!l1_desc->l2ptr) { - if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc)) + l1_desc = &cd_table->l1_desc[ssid / CTXDESC_L2_ENTRIES]; + if (!l1_desc->l2ptr) + return NULL; + return &l1_desc->l2ptr[ssid % CTXDESC_L2_ENTRIES]; +} + +struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, + u32 ssid) +{ + struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; + struct arm_smmu_device *smmu = master->smmu; + + might_sleep(); + iommu_group_mutex_assert(master->dev); + + if (!cd_table->cdtab) { + if (arm_smmu_alloc_cd_tables(master)) return NULL; + } + + if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_64K_L2) { + unsigned int idx = ssid / CTXDESC_L2_ENTRIES; + struct arm_smmu_l1_ctx_desc *l1_desc; + + l1_desc = &cd_table->l1_desc[idx]; + if (!l1_desc->l2ptr) { + __le64 *l1ptr; - l1ptr = cd_table->cdtab + idx * CTXDESC_L1_DESC_DWORDS; - arm_smmu_write_cd_l1_desc(l1ptr, l1_desc); - /* An invalid L1CD can be cached */ - arm_smmu_sync_cd(master, ssid, false); + if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc)) + return NULL; + + l1ptr = cd_table->cdtab + idx * CTXDESC_L1_DESC_DWORDS; + arm_smmu_write_cd_l1_desc(l1ptr, l1_desc); + /* An invalid L1CD can be cached */ + arm_smmu_sync_cd(master, ssid, false); + } } - idx = ssid & (CTXDESC_L2_ENTRIES - 1); - return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS; + return arm_smmu_get_cd_ptr(master, ssid); } -int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, - struct arm_smmu_ctx_desc *cd) +struct arm_smmu_cd_writer { + struct arm_smmu_entry_writer writer; + unsigned int ssid; +}; + +VISIBLE_IF_KUNIT +void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits) { + used_bits[0] = cpu_to_le64(CTXDESC_CD_0_V); + if (!(ent[0] & cpu_to_le64(CTXDESC_CD_0_V))) + return; + memset(used_bits, 0xFF, sizeof(struct arm_smmu_cd)); + /* - * This function handles the following cases: - * - * (1) Install primary CD, for normal DMA traffic (SSID = IOMMU_NO_PASID = 0). - * (2) Install a secondary CD, for SID+SSID traffic. - * (3) Update ASID of a CD. Atomically write the first 64 bits of the - * CD, then invalidate the old entry and mappings. - * (4) Quiesce the context without clearing the valid bit. Disable - * translation, and ignore any translation fault. - * (5) Remove a secondary CD. + * If EPD0 is set by the make function it means + * T0SZ/TG0/IR0/OR0/SH0/TTB0 are IGNORED */ - u64 val; - bool cd_live; - __le64 *cdptr; - struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; - struct arm_smmu_device *smmu = master->smmu; + if (ent[0] & cpu_to_le64(CTXDESC_CD_0_TCR_EPD0)) { + used_bits[0] &= ~cpu_to_le64( + CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 | + CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 | + CTXDESC_CD_0_TCR_SH0); + used_bits[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK); + } +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_cd_used); - if (WARN_ON(ssid >= (1 << cd_table->s1cdmax))) - return -E2BIG; +static void arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer *writer) +{ + struct arm_smmu_cd_writer *cd_writer = + container_of(writer, struct arm_smmu_cd_writer, writer); - cdptr = arm_smmu_get_cd_ptr(master, ssid); - if (!cdptr) - return -ENOMEM; + arm_smmu_sync_cd(writer->master, cd_writer->ssid, true); +} - val = le64_to_cpu(cdptr[0]); - cd_live = !!(val & CTXDESC_CD_0_V); - - if (!cd) { /* (5) */ - val = 0; - } else if (cd == &quiet_cd) { /* (4) */ - if (!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) - val &= ~(CTXDESC_CD_0_S | CTXDESC_CD_0_R); - val |= CTXDESC_CD_0_TCR_EPD0; - } else if (cd_live) { /* (3) */ - val &= ~CTXDESC_CD_0_ASID; - val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid); - /* - * Until CD+TLB invalidation, both ASIDs may be used for tagging - * this substream's traffic - */ - } else { /* (1) and (2) */ - cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); - cdptr[2] = 0; - cdptr[3] = cpu_to_le64(cd->mair); +static const struct arm_smmu_entry_writer_ops arm_smmu_cd_writer_ops = { + .sync = arm_smmu_cd_writer_sync_entry, + .get_used = arm_smmu_get_cd_used, +}; - /* - * STE may be live, and the SMMU might read dwords of this CD in any - * order. Ensure that it observes valid values before reading - * V=1. - */ - arm_smmu_sync_cd(master, ssid, true); +void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid, + struct arm_smmu_cd *cdptr, + const struct arm_smmu_cd *target) +{ + struct arm_smmu_cd_writer cd_writer = { + .writer = { + .ops = &arm_smmu_cd_writer_ops, + .master = master, + }, + .ssid = ssid, + }; + + arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data); +} - val = cd->tcr | +void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, + struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_ctx_desc *cd = &smmu_domain->cd; + const struct io_pgtable_cfg *pgtbl_cfg = + &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg; + typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = + &pgtbl_cfg->arm_lpae_s1_cfg.tcr; + + memset(target, 0, sizeof(*target)); + + target->data[0] = cpu_to_le64( + FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) | + FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) | + FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) | + FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) | + FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) | #ifdef __BIG_ENDIAN - CTXDESC_CD_0_ENDI | + CTXDESC_CD_0_ENDI | #endif - CTXDESC_CD_0_R | CTXDESC_CD_0_A | - (cd->mm ? 0 : CTXDESC_CD_0_ASET) | - CTXDESC_CD_0_AA64 | - FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) | - CTXDESC_CD_0_V; - - if (cd_table->stall_enabled) - val |= CTXDESC_CD_0_S; - } - - /* - * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3 - * "Configuration structures and configuration invalidation completion" - * - * The size of single-copy atomic reads made by the SMMU is - * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single - * field within an aligned 64-bit span of a structure can be altered - * without first making the structure invalid. - */ - WRITE_ONCE(cdptr[0], cpu_to_le64(val)); - arm_smmu_sync_cd(master, ssid, true); - return 0; + CTXDESC_CD_0_TCR_EPD1 | + CTXDESC_CD_0_V | + FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) | + CTXDESC_CD_0_AA64 | + (master->stall_enabled ? CTXDESC_CD_0_S : 0) | + CTXDESC_CD_0_R | + CTXDESC_CD_0_A | + CTXDESC_CD_0_ASET | + FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) + ); + target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr & + CTXDESC_CD_1_TTB0_MASK); + target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair); +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s1_cd); + +void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid) +{ + struct arm_smmu_cd target = {}; + struct arm_smmu_cd *cdptr; + + if (!master->cd_table.cdtab) + return; + cdptr = arm_smmu_get_cd_ptr(master, ssid); + if (WARN_ON(!cdptr)) + return; + arm_smmu_write_cd_entry(master, ssid, cdptr, &target); } static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master) @@ -1331,7 +1358,6 @@ static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master) struct arm_smmu_device *smmu = master->smmu; struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; - cd_table->stall_enabled = master->stall_enabled; cd_table->s1cdmax = master->ssid_bits; max_contexts = 1 << cd_table->s1cdmax; @@ -1429,33 +1455,75 @@ arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc) val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span); val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; - /* See comment in arm_smmu_write_ctx_desc() */ + /* The HW has 64 bit atomicity with stores to the L2 STE table */ WRITE_ONCE(*dst, cpu_to_le64(val)); } -static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) +struct arm_smmu_ste_writer { + struct arm_smmu_entry_writer writer; + u32 sid; +}; + +static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer) { + struct arm_smmu_ste_writer *ste_writer = + container_of(writer, struct arm_smmu_ste_writer, writer); struct arm_smmu_cmdq_ent cmd = { .opcode = CMDQ_OP_CFGI_STE, .cfgi = { - .sid = sid, + .sid = ste_writer->sid, .leaf = true, }, }; - arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); + arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd); } -static void arm_smmu_make_abort_ste(struct arm_smmu_ste *target) +static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = { + .sync = arm_smmu_ste_writer_sync_entry, + .get_used = arm_smmu_get_ste_used, +}; + +static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, + struct arm_smmu_ste *ste, + const struct arm_smmu_ste *target) +{ + struct arm_smmu_device *smmu = master->smmu; + struct arm_smmu_ste_writer ste_writer = { + .writer = { + .ops = &arm_smmu_ste_writer_ops, + .master = master, + }, + .sid = sid, + }; + + arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data); + + /* It's likely that we'll want to use the new STE soon */ + if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) { + struct arm_smmu_cmdq_ent + prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG, + .prefetch = { + .sid = sid, + } }; + + arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); + } +} + +VISIBLE_IF_KUNIT +void arm_smmu_make_abort_ste(struct arm_smmu_ste *target) { memset(target, 0, sizeof(*target)); target->data[0] = cpu_to_le64( STRTAB_STE_0_V | FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT)); } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_abort_ste); -static void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, - struct arm_smmu_ste *target) +VISIBLE_IF_KUNIT +void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, + struct arm_smmu_ste *target) { memset(target, 0, sizeof(*target)); target->data[0] = cpu_to_le64( @@ -1466,9 +1534,11 @@ static void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, target->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING)); } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_bypass_ste); -static void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, - struct arm_smmu_master *master) +VISIBLE_IF_KUNIT +void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master) { struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; struct arm_smmu_device *smmu = master->smmu; @@ -1516,10 +1586,12 @@ static void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, cpu_to_le64(FIELD_PREP(STRTAB_STE_2_S2VMID, 0)); } } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_cdtable_ste); -static void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, - struct arm_smmu_master *master, - struct arm_smmu_domain *smmu_domain) +VISIBLE_IF_KUNIT +void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain) { struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg; const struct io_pgtable_cfg *pgtbl_cfg = @@ -1562,22 +1634,19 @@ static void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr & STRTAB_STE_3_S2TTB_MASK); } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s2_domain_ste); /* * This can safely directly manipulate the STE memory without a sync sequence * because the STE table has not been installed in the SMMU yet. */ -static void arm_smmu_init_initial_stes(struct arm_smmu_device *smmu, - struct arm_smmu_ste *strtab, +static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab, unsigned int nent) { unsigned int i; for (i = 0; i < nent; ++i) { - if (disable_bypass) - arm_smmu_make_abort_ste(strtab); - else - arm_smmu_make_bypass_ste(smmu, strtab); + arm_smmu_make_abort_ste(strtab); strtab++; } } @@ -1605,7 +1674,7 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) return -ENOMEM; } - arm_smmu_init_initial_stes(smmu, desc->l2ptr, 1 << STRTAB_SPLIT); + arm_smmu_init_initial_stes(desc->l2ptr, 1 << STRTAB_SPLIT); arm_smmu_write_strtab_l1_desc(strtab, desc); return 0; } @@ -2230,13 +2299,11 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) } static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu, - struct arm_smmu_domain *smmu_domain, - struct io_pgtable_cfg *pgtbl_cfg) + struct arm_smmu_domain *smmu_domain) { int ret; u32 asid; struct arm_smmu_ctx_desc *cd = &smmu_domain->cd; - typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr; refcount_set(&cd->refs, 1); @@ -2244,31 +2311,13 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu, mutex_lock(&arm_smmu_asid_lock); ret = xa_alloc(&arm_smmu_asid_xa, &asid, cd, XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); - if (ret) - goto out_unlock; - cd->asid = (u16)asid; - cd->ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr; - cd->tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) | - FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) | - FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) | - FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) | - FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) | - FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) | - CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; - cd->mair = pgtbl_cfg->arm_lpae_s1_cfg.mair; - - mutex_unlock(&arm_smmu_asid_lock); - return 0; - -out_unlock: mutex_unlock(&arm_smmu_asid_lock); return ret; } static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu, - struct arm_smmu_domain *smmu_domain, - struct io_pgtable_cfg *pgtbl_cfg) + struct arm_smmu_domain *smmu_domain) { int vmid; struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; @@ -2292,8 +2341,7 @@ static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg pgtbl_cfg; struct io_pgtable_ops *pgtbl_ops; int (*finalise_stage_fn)(struct arm_smmu_device *smmu, - struct arm_smmu_domain *smmu_domain, - struct io_pgtable_cfg *pgtbl_cfg); + struct arm_smmu_domain *smmu_domain); /* Restrict the stage to what we can actually support */ if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) @@ -2336,7 +2384,7 @@ static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; smmu_domain->domain.geometry.force_aperture = true; - ret = finalise_stage_fn(smmu, smmu_domain, &pgtbl_cfg); + ret = finalise_stage_fn(smmu, smmu_domain); if (ret < 0) { free_io_pgtable_ops(pgtbl_ops); return ret; @@ -2419,7 +2467,10 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master, pdev = to_pci_dev(master->dev); atomic_inc(&smmu_domain->nr_ats_masters); - arm_smmu_atc_inv_domain(smmu_domain, IOMMU_NO_PASID, 0, 0); + /* + * ATC invalidation of PASID 0 causes the entire ATC to be flushed. + */ + arm_smmu_atc_inv_master(master); if (pci_enable_ats(pdev, stu)) dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu); } @@ -2515,6 +2566,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) struct arm_smmu_device *smmu; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_master *master; + struct arm_smmu_cd *cdptr; if (!fwspec) return -ENOENT; @@ -2543,6 +2595,12 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) if (ret) return ret; + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID); + if (!cdptr) + return -ENOMEM; + } + /* * Prevent arm_smmu_share_asid() from trying to change the ASID * of either the old or new domain while we are working on it. @@ -2560,49 +2618,26 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); switch (smmu_domain->stage) { - case ARM_SMMU_DOMAIN_S1: - if (!master->cd_table.cdtab) { - ret = arm_smmu_alloc_cd_tables(master); - if (ret) - goto out_list_del; - } else { - /* - * arm_smmu_write_ctx_desc() relies on the entry being - * invalid to work, clear any existing entry. - */ - ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, - NULL); - if (ret) - goto out_list_del; - } - - ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, &smmu_domain->cd); - if (ret) - goto out_list_del; + case ARM_SMMU_DOMAIN_S1: { + struct arm_smmu_cd target_cd; + arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); + arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr, + &target_cd); arm_smmu_make_cdtable_ste(&target, master); arm_smmu_install_ste_for_dev(master, &target); break; + } case ARM_SMMU_DOMAIN_S2: arm_smmu_make_s2_domain_ste(&target, master, smmu_domain); arm_smmu_install_ste_for_dev(master, &target); - if (master->cd_table.cdtab) - arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, - NULL); + arm_smmu_clear_cd(master, IOMMU_NO_PASID); break; } arm_smmu_enable_ats(master, smmu_domain); - goto out_unlock; - -out_list_del: - spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_del_init(&master->domain_head); - spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - -out_unlock: mutex_unlock(&arm_smmu_asid_lock); - return ret; + return 0; } static int arm_smmu_attach_dev_ste(struct device *dev, @@ -2636,8 +2671,7 @@ static int arm_smmu_attach_dev_ste(struct device *dev, * arm_smmu_domain->devices to avoid races updating the same context * descriptor from arm_smmu_share_asid(). */ - if (master->cd_table.cdtab) - arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); + arm_smmu_clear_cd(master, IOMMU_NO_PASID); return 0; } @@ -2915,10 +2949,10 @@ static void arm_smmu_release_device(struct device *dev) iopf_queue_remove_device(master->smmu->evtq.iopf, dev); /* Put the STE back to what arm_smmu_init_strtab() sets */ - if (disable_bypass && !dev->iommu->require_direct) - arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev); - else + if (dev->iommu->require_direct) arm_smmu_attach_dev_identity(&arm_smmu_identity_domain, dev); + else + arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev); arm_smmu_disable_pasid(master); arm_smmu_remove_master(master); @@ -3053,14 +3087,9 @@ static int arm_smmu_def_domain_type(struct device *dev) return 0; } -static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid) +static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid, + struct iommu_domain *domain) { - struct iommu_domain *domain; - - domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA); - if (WARN_ON(IS_ERR(domain)) || !domain) - return; - arm_smmu_sva_remove_dev_pasid(domain, dev, pasid); } @@ -3273,7 +3302,7 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits); cfg->strtab_base_cfg = reg; - arm_smmu_init_initial_stes(smmu, strtab, cfg->num_l1_ents); + arm_smmu_init_initial_stes(strtab, cfg->num_l1_ents); return 0; } @@ -3402,7 +3431,7 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX); /* Add callback to free MSIs on teardown */ - devm_add_action(dev, arm_smmu_free_msis, dev); + devm_add_action_or_reset(dev, arm_smmu_free_msis, dev); } static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) @@ -3503,7 +3532,7 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu) return ret; } -static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) +static int arm_smmu_device_reset(struct arm_smmu_device *smmu) { int ret; u32 reg, enables; @@ -3513,7 +3542,6 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); if (reg & CR0_SMMUEN) { dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); - WARN_ON(is_kdump_kernel() && !disable_bypass); arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0); } @@ -3620,14 +3648,8 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) if (is_kdump_kernel()) enables &= ~(CR0_EVTQEN | CR0_PRIQEN); - /* Enable the SMMU interface, or ensure bypass */ - if (!bypass || disable_bypass) { - enables |= CR0_SMMUEN; - } else { - ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT); - if (ret) - return ret; - } + /* Enable the SMMU interface */ + enables |= CR0_SMMUEN; ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); if (ret) { @@ -4019,7 +4041,6 @@ static int arm_smmu_device_probe(struct platform_device *pdev) resource_size_t ioaddr; struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; - bool bypass; smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) @@ -4030,12 +4051,9 @@ static int arm_smmu_device_probe(struct platform_device *pdev) ret = arm_smmu_device_dt_probe(pdev, smmu); } else { ret = arm_smmu_device_acpi_probe(pdev, smmu); - if (ret == -ENODEV) - return ret; } - - /* Set bypass mode according to firmware probing result */ - bypass = !!ret; + if (ret) + return ret; /* Base address */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -4099,7 +4117,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) arm_smmu_rmr_install_bypass_ste(smmu); /* Reset the device */ - ret = arm_smmu_device_reset(smmu, bypass); + ret = arm_smmu_device_reset(smmu); if (ret) return ret; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 2a19bb63e5c6..1242a086c9f9 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -275,14 +275,18 @@ struct arm_smmu_ste { * 2lvl: at most 1024 L1 entries, * 1024 lazy entries per table. */ -#define CTXDESC_SPLIT 10 -#define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT) +#define CTXDESC_L2_ENTRIES 1024 #define CTXDESC_L1_DESC_DWORDS 1 #define CTXDESC_L1_DESC_V (1UL << 0) #define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12) #define CTXDESC_CD_DWORDS 8 + +struct arm_smmu_cd { + __le64 data[CTXDESC_CD_DWORDS]; +}; + #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6) #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) @@ -583,16 +587,13 @@ struct arm_smmu_strtab_l1_desc { struct arm_smmu_ctx_desc { u16 asid; - u64 ttbr; - u64 tcr; - u64 mair; refcount_t refs; struct mm_struct *mm; }; struct arm_smmu_l1_ctx_desc { - __le64 *l2ptr; + struct arm_smmu_cd *l2ptr; dma_addr_t l2ptr_dma; }; @@ -604,8 +605,6 @@ struct arm_smmu_ctx_desc_cfg { u8 s1fmt; /* log2 of the maximum number of CDs supported by this table */ u8 s1cdmax; - /* Whether CD entries in this table have the stall bit set. */ - u8 stall_enabled:1; }; struct arm_smmu_s2_cfg { @@ -737,6 +736,36 @@ struct arm_smmu_domain { struct list_head mmu_notifiers; }; +/* The following are exposed for testing purposes. */ +struct arm_smmu_entry_writer_ops; +struct arm_smmu_entry_writer { + const struct arm_smmu_entry_writer_ops *ops; + struct arm_smmu_master *master; +}; + +struct arm_smmu_entry_writer_ops { + void (*get_used)(const __le64 *entry, __le64 *used); + void (*sync)(struct arm_smmu_entry_writer *writer); +}; + +#if IS_ENABLED(CONFIG_KUNIT) +void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits); +void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur, + const __le64 *target); +void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits); +void arm_smmu_make_abort_ste(struct arm_smmu_ste *target); +void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, + struct arm_smmu_ste *target); +void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master); +void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain); +void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, + struct arm_smmu_master *master, struct mm_struct *mm, + u16 asid); +#endif + static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) { return container_of(dom, struct arm_smmu_domain, domain); @@ -744,10 +773,19 @@ static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) extern struct xarray arm_smmu_asid_xa; extern struct mutex arm_smmu_asid_lock; -extern struct arm_smmu_ctx_desc quiet_cd; -int arm_smmu_write_ctx_desc(struct arm_smmu_master *smmu_master, int ssid, - struct arm_smmu_ctx_desc *cd); +void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid); +struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, + u32 ssid); +struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, + u32 ssid); +void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, + struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain); +void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid, + struct arm_smmu_cd *cdptr, + const struct arm_smmu_cd *target); + void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid); void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, size_t granule, bool leaf, diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c index bb89d49adf8d..552199cbd9e2 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c @@ -1,15 +1,66 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. */ +#include <linux/cleanup.h> #include <linux/device.h> +#include <linux/interconnect.h> #include <linux/firmware/qcom/qcom_scm.h> +#include <linux/iopoll.h> +#include <linux/list.h> +#include <linux/mod_devicetable.h> +#include <linux/mutex.h> +#include <linux/platform_device.h> #include <linux/ratelimit.h> +#include <linux/spinlock.h> #include "arm-smmu.h" #include "arm-smmu-qcom.h" +#define TBU_DBG_TIMEOUT_US 100 +#define DEBUG_AXUSER_REG 0x30 +#define DEBUG_AXUSER_CDMID GENMASK_ULL(43, 36) +#define DEBUG_AXUSER_CDMID_VAL 0xff +#define DEBUG_PAR_REG 0x28 +#define DEBUG_PAR_FAULT_VAL BIT(0) +#define DEBUG_PAR_PA GENMASK_ULL(47, 12) +#define DEBUG_SID_HALT_REG 0x0 +#define DEBUG_SID_HALT_VAL BIT(16) +#define DEBUG_SID_HALT_SID GENMASK(9, 0) +#define DEBUG_SR_HALT_ACK_REG 0x20 +#define DEBUG_SR_HALT_ACK_VAL BIT(1) +#define DEBUG_SR_ECATS_RUNNING_VAL BIT(0) +#define DEBUG_TXN_AXCACHE GENMASK(5, 2) +#define DEBUG_TXN_AXPROT GENMASK(8, 6) +#define DEBUG_TXN_AXPROT_PRIV 0x1 +#define DEBUG_TXN_AXPROT_NSEC 0x2 +#define DEBUG_TXN_TRIGG_REG 0x18 +#define DEBUG_TXN_TRIGGER BIT(0) +#define DEBUG_VA_ADDR_REG 0x8 + +static LIST_HEAD(tbu_list); +static DEFINE_MUTEX(tbu_list_lock); +static DEFINE_SPINLOCK(atos_lock); + +struct qcom_tbu { + struct device *dev; + struct device_node *smmu_np; + u32 sid_range[2]; + struct list_head list; + struct clk *clk; + struct icc_path *path; + void __iomem *base; + spinlock_t halt_lock; /* multiple halt or resume can't execute concurrently */ + int halt_count; +}; + +static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu) +{ + return container_of(smmu, struct qcom_smmu, smmu); +} + void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu) { int ret; @@ -49,3 +100,448 @@ void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu) tbu_pwr_status, sync_inv_ack, sync_inv_progress); } } + +static struct qcom_tbu *qcom_find_tbu(struct qcom_smmu *qsmmu, u32 sid) +{ + struct qcom_tbu *tbu; + u32 start, end; + + guard(mutex)(&tbu_list_lock); + + if (list_empty(&tbu_list)) + return NULL; + + list_for_each_entry(tbu, &tbu_list, list) { + start = tbu->sid_range[0]; + end = start + tbu->sid_range[1]; + + if (qsmmu->smmu.dev->of_node == tbu->smmu_np && + start <= sid && sid < end) + return tbu; + } + dev_err(qsmmu->smmu.dev, "Unable to find TBU for sid 0x%x\n", sid); + + return NULL; +} + +static int qcom_tbu_halt(struct qcom_tbu *tbu, struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + int ret = 0, idx = smmu_domain->cfg.cbndx; + u32 val, fsr, status; + + guard(spinlock_irqsave)(&tbu->halt_lock); + if (tbu->halt_count) { + tbu->halt_count++; + return ret; + } + + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val |= DEBUG_SID_HALT_VAL; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if ((fsr & ARM_SMMU_FSR_FAULT) && (fsr & ARM_SMMU_FSR_SS)) { + u32 sctlr_orig, sctlr; + + /* + * We are in a fault. Our request to halt the bus will not + * complete until transactions in front of us (such as the fault + * itself) have completed. Disable iommu faults and terminate + * any existing transactions. + */ + sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(ARM_SMMU_SCTLR_CFCFG | ARM_SMMU_SCTLR_CFIE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, ARM_SMMU_RESUME_TERMINATE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); + } + + if (readl_poll_timeout_atomic(tbu->base + DEBUG_SR_HALT_ACK_REG, status, + (status & DEBUG_SR_HALT_ACK_VAL), + 0, TBU_DBG_TIMEOUT_US)) { + dev_err(tbu->dev, "Timeout while trying to halt TBU!\n"); + ret = -ETIMEDOUT; + + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_VAL; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + return ret; + } + + tbu->halt_count = 1; + + return ret; +} + +static void qcom_tbu_resume(struct qcom_tbu *tbu) +{ + u32 val; + + guard(spinlock_irqsave)(&tbu->halt_lock); + if (!tbu->halt_count) { + WARN(1, "%s: halt_count is 0", dev_name(tbu->dev)); + return; + } + + if (tbu->halt_count > 1) { + tbu->halt_count--; + return; + } + + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_VAL; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + tbu->halt_count = 0; +} + +static phys_addr_t qcom_tbu_trigger_atos(struct arm_smmu_domain *smmu_domain, + struct qcom_tbu *tbu, dma_addr_t iova, u32 sid) +{ + bool atos_timedout = false; + phys_addr_t phys = 0; + ktime_t timeout; + u64 val; + + /* Set address and stream-id */ + val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID; + val |= FIELD_PREP(DEBUG_SID_HALT_SID, sid); + writeq_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + writeq_relaxed(iova, tbu->base + DEBUG_VA_ADDR_REG); + val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL); + writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG); + + /* Write-back read and write-allocate */ + val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xf); + + /* Non-secure access */ + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC); + + /* Privileged access */ + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV); + + val |= DEBUG_TXN_TRIGGER; + writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG); + + timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US); + for (;;) { + val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); + if (!(val & DEBUG_SR_ECATS_RUNNING_VAL)) + break; + val = readl_relaxed(tbu->base + DEBUG_PAR_REG); + if (val & DEBUG_PAR_FAULT_VAL) + break; + if (ktime_compare(ktime_get(), timeout) > 0) { + atos_timedout = true; + break; + } + } + + val = readq_relaxed(tbu->base + DEBUG_PAR_REG); + if (val & DEBUG_PAR_FAULT_VAL) + dev_err(tbu->dev, "ATOS generated a fault interrupt! PAR = %llx, SID=0x%x\n", + val, sid); + else if (atos_timedout) + dev_err_ratelimited(tbu->dev, "ATOS translation timed out!\n"); + else + phys = FIELD_GET(DEBUG_PAR_PA, val); + + /* Reset hardware */ + writeq_relaxed(0, tbu->base + DEBUG_TXN_TRIGG_REG); + writeq_relaxed(0, tbu->base + DEBUG_VA_ADDR_REG); + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + return phys; +} + +static phys_addr_t qcom_iova_to_phys(struct arm_smmu_domain *smmu_domain, + dma_addr_t iova, u32 sid) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); + int idx = smmu_domain->cfg.cbndx; + struct qcom_tbu *tbu; + u32 sctlr_orig, sctlr; + phys_addr_t phys = 0; + int attempt = 0; + int ret; + u64 fsr; + + tbu = qcom_find_tbu(qsmmu, sid); + if (!tbu) + return 0; + + ret = icc_set_bw(tbu->path, 0, UINT_MAX); + if (ret) + return ret; + + ret = clk_prepare_enable(tbu->clk); + if (ret) + goto disable_icc; + + ret = qcom_tbu_halt(tbu, smmu_domain); + if (ret) + goto disable_clk; + + /* + * ATOS/ECATS can trigger the fault interrupt, so disable it temporarily + * and check for an interrupt manually. + */ + sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(ARM_SMMU_SCTLR_CFCFG | ARM_SMMU_SCTLR_CFIE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (fsr & ARM_SMMU_FSR_FAULT) { + /* Clear pending interrupts */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + + /* + * TBU halt takes care of resuming any stalled transcation. + * Kept it here for completeness sake. + */ + if (fsr & ARM_SMMU_FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + ARM_SMMU_RESUME_TERMINATE); + } + + /* Only one concurrent atos operation */ + scoped_guard(spinlock_irqsave, &atos_lock) { + /* + * If the translation fails, attempt the lookup more time." + */ + do { + phys = qcom_tbu_trigger_atos(smmu_domain, tbu, iova, sid); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (fsr & ARM_SMMU_FSR_FAULT) { + /* Clear pending interrupts */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + + if (fsr & ARM_SMMU_FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + ARM_SMMU_RESUME_TERMINATE); + } + } while (!phys && attempt++ < 2); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); + } + qcom_tbu_resume(tbu); + + /* Read to complete prior write transcations */ + readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); + +disable_clk: + clk_disable_unprepare(tbu->clk); +disable_icc: + icc_set_bw(tbu->path, 0, 0); + + return phys; +} + +static phys_addr_t qcom_smmu_iova_to_phys_hard(struct arm_smmu_domain *smmu_domain, dma_addr_t iova) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + int idx = smmu_domain->cfg.cbndx; + u32 frsynra; + u16 sid; + + frsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); + sid = FIELD_GET(ARM_SMMU_CBFRSYNRA_SID, frsynra); + + return qcom_iova_to_phys(smmu_domain, iova, sid); +} + +static phys_addr_t qcom_smmu_verify_fault(struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 fsr) +{ + struct io_pgtable *iop = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops); + struct arm_smmu_device *smmu = smmu_domain->smmu; + phys_addr_t phys_post_tlbiall; + phys_addr_t phys; + + phys = qcom_smmu_iova_to_phys_hard(smmu_domain, iova); + io_pgtable_tlb_flush_all(iop); + phys_post_tlbiall = qcom_smmu_iova_to_phys_hard(smmu_domain, iova); + + if (phys != phys_post_tlbiall) { + dev_err(smmu->dev, + "ATOS results differed across TLBIALL... (before: %pa after: %pa)\n", + &phys, &phys_post_tlbiall); + } + + return (phys == 0 ? phys_post_tlbiall : phys); +} + +irqreturn_t qcom_smmu_context_fault(int irq, void *dev) +{ + struct arm_smmu_domain *smmu_domain = dev; + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + struct arm_smmu_device *smmu = smmu_domain->smmu; + u32 fsr, fsynr, cbfrsynra, resume = 0; + int idx = smmu_domain->cfg.cbndx; + phys_addr_t phys_soft; + unsigned long iova; + int ret, tmp; + + static DEFINE_RATELIMIT_STATE(_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (!(fsr & ARM_SMMU_FSR_FAULT)) + return IRQ_NONE; + + fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); + iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); + cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); + + if (list_empty(&tbu_list)) { + ret = report_iommu_fault(&smmu_domain->domain, NULL, iova, + fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); + + if (ret == -ENOSYS) + dev_err_ratelimited(smmu->dev, + "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", + fsr, iova, fsynr, cbfrsynra, idx); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + return IRQ_HANDLED; + } + + phys_soft = ops->iova_to_phys(ops, iova); + + tmp = report_iommu_fault(&smmu_domain->domain, NULL, iova, + fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); + if (!tmp || tmp == -EBUSY) { + dev_dbg(smmu->dev, + "Context fault handled by client: iova=0x%08lx, fsr=0x%x, fsynr=0x%x, cb=%d\n", + iova, fsr, fsynr, idx); + dev_dbg(smmu->dev, "soft iova-to-phys=%pa\n", &phys_soft); + ret = IRQ_HANDLED; + resume = ARM_SMMU_RESUME_TERMINATE; + } else { + phys_addr_t phys_atos = qcom_smmu_verify_fault(smmu_domain, iova, fsr); + + if (__ratelimit(&_rs)) { + dev_err(smmu->dev, + "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", + fsr, iova, fsynr, cbfrsynra, idx); + dev_err(smmu->dev, + "FSR = %08x [%s%s%s%s%s%s%s%s%s], SID=0x%x\n", + fsr, + (fsr & 0x02) ? "TF " : "", + (fsr & 0x04) ? "AFF " : "", + (fsr & 0x08) ? "PF " : "", + (fsr & 0x10) ? "EF " : "", + (fsr & 0x20) ? "TLBMCF " : "", + (fsr & 0x40) ? "TLBLKF " : "", + (fsr & 0x80) ? "MHF " : "", + (fsr & 0x40000000) ? "SS " : "", + (fsr & 0x80000000) ? "MULTI " : "", + cbfrsynra); + + dev_err(smmu->dev, + "soft iova-to-phys=%pa\n", &phys_soft); + if (!phys_soft) + dev_err(smmu->dev, + "SOFTWARE TABLE WALK FAILED! Looks like %s accessed an unmapped address!\n", + dev_name(smmu->dev)); + if (phys_atos) + dev_err(smmu->dev, "hard iova-to-phys (ATOS)=%pa\n", + &phys_atos); + else + dev_err(smmu->dev, "hard iova-to-phys (ATOS) failed\n"); + } + ret = IRQ_NONE; + resume = ARM_SMMU_RESUME_TERMINATE; + } + + /* + * If the client returns -EBUSY, do not clear FSR and do not RESUME + * if stalled. This is required to keep the IOMMU client stalled on + * the outstanding fault. This gives the client a chance to take any + * debug action and then terminate the stalled transaction. + * So, the sequence in case of stall on fault should be: + * 1) Do not clear FSR or write to RESUME here + * 2) Client takes any debug action + * 3) Client terminates the stalled transaction and resumes the IOMMU + * 4) Client clears FSR. The FSR should only be cleared after 3) and + * not before so that the fault remains outstanding. This ensures + * SCTLR.HUPCF has the desired effect if subsequent transactions also + * need to be terminated. + */ + if (tmp != -EBUSY) { + /* Clear the faulting FSR */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + + /* Retry or terminate any stalled transactions */ + if (fsr & ARM_SMMU_FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, resume); + } + + return ret; +} + +static int qcom_tbu_probe(struct platform_device *pdev) +{ + struct of_phandle_args args = { .args_count = 2 }; + struct device_node *np = pdev->dev.of_node; + struct device *dev = &pdev->dev; + struct qcom_tbu *tbu; + + tbu = devm_kzalloc(dev, sizeof(*tbu), GFP_KERNEL); + if (!tbu) + return -ENOMEM; + + tbu->dev = dev; + INIT_LIST_HEAD(&tbu->list); + spin_lock_init(&tbu->halt_lock); + + if (of_parse_phandle_with_args(np, "qcom,stream-id-range", "#iommu-cells", 0, &args)) { + dev_err(dev, "Cannot parse the 'qcom,stream-id-range' DT property\n"); + return -EINVAL; + } + + tbu->smmu_np = args.np; + tbu->sid_range[0] = args.args[0]; + tbu->sid_range[1] = args.args[1]; + of_node_put(args.np); + + tbu->base = devm_of_iomap(dev, np, 0, NULL); + if (IS_ERR(tbu->base)) + return PTR_ERR(tbu->base); + + tbu->clk = devm_clk_get_optional(dev, NULL); + if (IS_ERR(tbu->clk)) + return PTR_ERR(tbu->clk); + + tbu->path = devm_of_icc_get(dev, NULL); + if (IS_ERR(tbu->path)) + return PTR_ERR(tbu->path); + + guard(mutex)(&tbu_list_lock); + list_add_tail(&tbu->list, &tbu_list); + + return 0; +} + +static const struct of_device_id qcom_tbu_of_match[] = { + { .compatible = "qcom,sc7280-tbu" }, + { .compatible = "qcom,sdm845-tbu" }, + { } +}; + +static struct platform_driver qcom_tbu_driver = { + .driver = { + .name = "qcom_tbu", + .of_match_table = qcom_tbu_of_match, + }, + .probe = qcom_tbu_probe, +}; +builtin_platform_driver(qcom_tbu_driver); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 5c7cfc51b57c..25f034677f56 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -413,6 +413,10 @@ static const struct arm_smmu_impl qcom_smmu_500_impl = { .reset = arm_mmu500_reset, .write_s2cr = qcom_smmu_write_s2cr, .tlb_sync = qcom_smmu_tlb_sync, +#ifdef CONFIG_ARM_SMMU_QCOM_DEBUG + .context_fault = qcom_smmu_context_fault, + .context_fault_needs_threaded_irq = true, +#endif }; static const struct arm_smmu_impl sdm845_smmu_500_impl = { @@ -422,6 +426,10 @@ static const struct arm_smmu_impl sdm845_smmu_500_impl = { .reset = qcom_sdm845_smmu500_reset, .write_s2cr = qcom_smmu_write_s2cr, .tlb_sync = qcom_smmu_tlb_sync, +#ifdef CONFIG_ARM_SMMU_QCOM_DEBUG + .context_fault = qcom_smmu_context_fault, + .context_fault_needs_threaded_irq = true, +#endif }; static const struct arm_smmu_impl qcom_adreno_smmu_v2_impl = { diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h index 593910567b88..9bb3ae7d62da 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h @@ -30,6 +30,8 @@ struct qcom_smmu_match_data { const struct arm_smmu_impl *adreno_impl; }; +irqreturn_t qcom_smmu_context_fault(int irq, void *dev); + #ifdef CONFIG_ARM_SMMU_QCOM_DEBUG void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu); #else diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index c572d877b0e1..87c81f75cf84 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -806,8 +806,16 @@ static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain, else context_fault = arm_smmu_context_fault; - ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED, - "arm-smmu-context-fault", smmu_domain); + if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq) + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, + context_fault, + IRQF_ONESHOT | IRQF_SHARED, + "arm-smmu-context-fault", + smmu_domain); + else + ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED, + "arm-smmu-context-fault", smmu_domain); + if (ret < 0) { dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", cfg->irptndx, irq); @@ -859,14 +867,10 @@ static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain) arm_smmu_rpm_put(smmu); } -static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) +static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev) { struct arm_smmu_domain *smmu_domain; - if (type != IOMMU_DOMAIN_UNMANAGED) { - if (using_legacy_binding || type != IOMMU_DOMAIN_DMA) - return NULL; - } /* * Allocate the domain and initialise some of its data structures. * We can't really do anything meaningful until we've added a @@ -1596,7 +1600,7 @@ static struct iommu_ops arm_smmu_ops = { .identity_domain = &arm_smmu_identity_domain, .blocked_domain = &arm_smmu_blocked_domain, .capable = arm_smmu_capable, - .domain_alloc = arm_smmu_domain_alloc, + .domain_alloc_paging = arm_smmu_domain_alloc_paging, .probe_device = arm_smmu_probe_device, .release_device = arm_smmu_release_device, .probe_finalize = arm_smmu_probe_finalize, diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h index 836ed6799a80..4765c6945c34 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h @@ -136,6 +136,7 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CBAR_VMID GENMASK(7, 0) #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2)) +#define ARM_SMMU_CBFRSYNRA_SID GENMASK(15, 0) #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) #define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16) @@ -238,6 +239,7 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CB_ATSR 0x8f0 #define ARM_SMMU_ATSR_ACTIVE BIT(0) +#define ARM_SMMU_RESUME_TERMINATE BIT(0) /* Maximum number of context banks per SMMU */ #define ARM_SMMU_MAX_CBS 128 @@ -436,6 +438,7 @@ struct arm_smmu_impl { int (*def_domain_type)(struct device *dev); irqreturn_t (*global_fault)(int irq, void *dev); irqreturn_t (*context_fault)(int irq, void *dev); + bool context_fault_needs_threaded_irq; int (*alloc_context_bank)(struct arm_smmu_domain *smmu_domain, struct arm_smmu_device *smmu, struct device *dev, int start); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index e4cb26f6a943..f731e4b2a417 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -32,6 +32,7 @@ #include <trace/events/swiotlb.h> #include "dma-iommu.h" +#include "iommu-pages.h" struct iommu_dma_msi_page { struct list_head list; @@ -156,7 +157,7 @@ static void fq_ring_free_locked(struct iommu_dma_cookie *cookie, struct iova_fq if (fq->entries[idx].counter >= counter) break; - put_pages_list(&fq->entries[idx].freelist); + iommu_put_pages_list(&fq->entries[idx].freelist); free_iova_fast(&cookie->iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages); @@ -254,7 +255,7 @@ static void iommu_dma_free_fq_single(struct iova_fq *fq) int idx; fq_ring_for_each(idx, fq) - put_pages_list(&fq->entries[idx].freelist); + iommu_put_pages_list(&fq->entries[idx].freelist); vfree(fq); } @@ -267,7 +268,7 @@ static void iommu_dma_free_fq_percpu(struct iova_fq __percpu *percpu_fq) struct iova_fq *fq = per_cpu_ptr(percpu_fq, cpu); fq_ring_for_each(idx, fq) - put_pages_list(&fq->entries[idx].freelist); + iommu_put_pages_list(&fq->entries[idx].freelist); } free_percpu(percpu_fq); @@ -660,19 +661,16 @@ static void iommu_dma_init_options(struct iommu_dma_options *options, /** * iommu_dma_init_domain - Initialise a DMA mapping domain * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() - * @base: IOVA at which the mappable address space starts - * @limit: Last address of the IOVA space * @dev: Device the domain is being initialised for * - * @base and @limit + 1 should be exact multiples of IOMMU page granularity to - * avoid rounding surprises. If necessary, we reserve the page at address 0 + * If the geometry and dma_range_map include address 0, we reserve that page * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but * any change which could make prior IOVAs invalid will fail. */ -static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, - dma_addr_t limit, struct device *dev) +static int iommu_dma_init_domain(struct iommu_domain *domain, struct device *dev) { struct iommu_dma_cookie *cookie = domain->iova_cookie; + const struct bus_dma_region *map = dev->dma_range_map; unsigned long order, base_pfn; struct iova_domain *iovad; int ret; @@ -684,18 +682,18 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, /* Use the smallest supported page size for IOVA granularity */ order = __ffs(domain->pgsize_bitmap); - base_pfn = max_t(unsigned long, 1, base >> order); + base_pfn = 1; /* Check the domain allows at least some access to the device... */ - if (domain->geometry.force_aperture) { + if (map) { + dma_addr_t base = dma_range_map_min(map); if (base > domain->geometry.aperture_end || - limit < domain->geometry.aperture_start) { + dma_range_map_max(map) < domain->geometry.aperture_start) { pr_warn("specified DMA range outside IOMMU capability\n"); return -EFAULT; } /* ...then finally give it a kicking to make sure it fits */ - base_pfn = max_t(unsigned long, base_pfn, - domain->geometry.aperture_start >> order); + base_pfn = max(base, domain->geometry.aperture_start) >> order; } /* start_pfn is always nonzero for an already-initialised domain */ @@ -1154,9 +1152,6 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, */ if (dev_use_swiotlb(dev, size, dir) && iova_offset(iovad, phys | size)) { - void *padding_start; - size_t padding_size, aligned_size; - if (!is_swiotlb_active(dev)) { dev_warn_once(dev, "DMA bounce buffers are inactive, unable to map unaligned transaction.\n"); return DMA_MAPPING_ERROR; @@ -1164,24 +1159,30 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, trace_swiotlb_bounced(dev, phys, size); - aligned_size = iova_align(iovad, size); - phys = swiotlb_tbl_map_single(dev, phys, size, aligned_size, + phys = swiotlb_tbl_map_single(dev, phys, size, iova_mask(iovad), dir, attrs); if (phys == DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; - /* Cleanup the padding area. */ - padding_start = phys_to_virt(phys); - padding_size = aligned_size; + /* + * Untrusted devices should not see padding areas with random + * leftover kernel data, so zero the pre- and post-padding. + * swiotlb_tbl_map_single() has initialized the bounce buffer + * proper to the contents of the original memory buffer. + */ + if (dev_is_untrusted(dev)) { + size_t start, virt = (size_t)phys_to_virt(phys); - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && - (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) { - padding_start += size; - padding_size -= size; - } + /* Pre-padding */ + start = iova_align_down(iovad, virt); + memset((void *)start, 0, virt - start); - memset(padding_start, 0, padding_size); + /* Post-padding */ + start = virt + size; + memset((void *)start, 0, + iova_align(iovad, start) - start); + } } if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) @@ -1720,10 +1721,11 @@ static size_t iommu_dma_max_mapping_size(struct device *dev) } static const struct dma_map_ops iommu_dma_ops = { - .flags = DMA_F_PCI_P2PDMA_SUPPORTED, + .flags = DMA_F_PCI_P2PDMA_SUPPORTED | + DMA_F_CAN_SKIP_SYNC, .alloc = iommu_dma_alloc, .free = iommu_dma_free, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, .alloc_noncontiguous = iommu_dma_alloc_noncontiguous, .free_noncontiguous = iommu_dma_free_noncontiguous, @@ -1744,25 +1746,20 @@ static const struct dma_map_ops iommu_dma_ops = { .max_mapping_size = iommu_dma_max_mapping_size, }; -/* - * The IOMMU core code allocates the default DMA domain, which the underlying - * IOMMU driver needs to support via the dma-iommu layer. - */ -void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) +void iommu_setup_dma_ops(struct device *dev) { struct iommu_domain *domain = iommu_get_domain_for_dev(dev); - if (!domain) - goto out_err; + if (dev_is_pci(dev)) + dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac; - /* - * The IOMMU core code allocates the default DMA domain, which the - * underlying IOMMU driver needs to support via the dma-iommu layer. - */ if (iommu_is_dma_domain(domain)) { - if (iommu_dma_init_domain(domain, dma_base, dma_limit, dev)) + if (iommu_dma_init_domain(domain, dev)) goto out_err; dev->dma_ops = &iommu_dma_ops; + } else if (dev->dma_ops == &iommu_dma_ops) { + /* Clean up if we've switched *from* a DMA domain */ + dev->dma_ops = NULL; } return; @@ -1770,7 +1767,6 @@ out_err: pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", dev_name(dev)); } -EXPORT_SYMBOL_GPL(iommu_setup_dma_ops); static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, phys_addr_t msi_addr, struct iommu_domain *domain) diff --git a/drivers/iommu/dma-iommu.h b/drivers/iommu/dma-iommu.h index c829f1f82a99..c12d63457c76 100644 --- a/drivers/iommu/dma-iommu.h +++ b/drivers/iommu/dma-iommu.h @@ -9,6 +9,8 @@ #ifdef CONFIG_IOMMU_DMA +void iommu_setup_dma_ops(struct device *dev); + int iommu_get_dma_cookie(struct iommu_domain *domain); void iommu_put_dma_cookie(struct iommu_domain *domain); @@ -17,13 +19,13 @@ int iommu_dma_init_fq(struct iommu_domain *domain); void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list); extern bool iommu_dma_forcedac; -static inline void iommu_dma_set_pci_32bit_workaround(struct device *dev) -{ - dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac; -} #else /* CONFIG_IOMMU_DMA */ +static inline void iommu_setup_dma_ops(struct device *dev) +{ +} + static inline int iommu_dma_init_fq(struct iommu_domain *domain) { return -EINVAL; @@ -42,9 +44,5 @@ static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_he { } -static inline void iommu_dma_set_pci_32bit_workaround(struct device *dev) -{ -} - #endif /* CONFIG_IOMMU_DMA */ #endif /* __DMA_IOMMU_H */ diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index d98c9161948a..c666ecab955d 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -22,6 +22,8 @@ #include <linux/pm_runtime.h> #include <linux/slab.h> +#include "iommu-pages.h" + typedef u32 sysmmu_iova_t; typedef u32 sysmmu_pte_t; static struct iommu_domain exynos_identity_domain; @@ -900,11 +902,11 @@ static struct iommu_domain *exynos_iommu_domain_alloc_paging(struct device *dev) if (!domain) return NULL; - domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); + domain->pgtable = iommu_alloc_pages(GFP_KERNEL, 2); if (!domain->pgtable) goto err_pgtable; - domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); + domain->lv2entcnt = iommu_alloc_pages(GFP_KERNEL, 1); if (!domain->lv2entcnt) goto err_counter; @@ -930,9 +932,9 @@ static struct iommu_domain *exynos_iommu_domain_alloc_paging(struct device *dev) return &domain->domain; err_lv2ent: - free_pages((unsigned long)domain->lv2entcnt, 1); + iommu_free_pages(domain->lv2entcnt, 1); err_counter: - free_pages((unsigned long)domain->pgtable, 2); + iommu_free_pages(domain->pgtable, 2); err_pgtable: kfree(domain); return NULL; @@ -973,8 +975,8 @@ static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) phys_to_virt(base)); } - free_pages((unsigned long)domain->pgtable, 2); - free_pages((unsigned long)domain->lv2entcnt, 1); + iommu_free_pages(domain->pgtable, 2); + iommu_free_pages(domain->lv2entcnt, 1); kfree(domain); } diff --git a/drivers/iommu/intel/Makefile b/drivers/iommu/intel/Makefile index 5402b699a122..c8beb0281559 100644 --- a/drivers/iommu/intel/Makefile +++ b/drivers/iommu/intel/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DMAR_TABLE) += dmar.o -obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o nested.o +obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o nested.o cache.o obj-$(CONFIG_DMAR_TABLE) += trace.o cap_audit.o obj-$(CONFIG_DMAR_PERF) += perf.o obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c new file mode 100644 index 000000000000..e8418cdd8331 --- /dev/null +++ b/drivers/iommu/intel/cache.c @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * cache.c - Intel VT-d cache invalidation + * + * Copyright (C) 2024 Intel Corporation + * + * Author: Lu Baolu <baolu.lu@linux.intel.com> + */ + +#define pr_fmt(fmt) "DMAR: " fmt + +#include <linux/dmar.h> +#include <linux/iommu.h> +#include <linux/memory.h> +#include <linux/pci.h> +#include <linux/spinlock.h> + +#include "iommu.h" +#include "pasid.h" +#include "trace.h" + +/* Check if an existing cache tag can be reused for a new association. */ +static bool cache_tage_match(struct cache_tag *tag, u16 domain_id, + struct intel_iommu *iommu, struct device *dev, + ioasid_t pasid, enum cache_tag_type type) +{ + if (tag->type != type) + return false; + + if (tag->domain_id != domain_id || tag->pasid != pasid) + return false; + + if (type == CACHE_TAG_IOTLB || type == CACHE_TAG_NESTING_IOTLB) + return tag->iommu == iommu; + + if (type == CACHE_TAG_DEVTLB || type == CACHE_TAG_NESTING_DEVTLB) + return tag->dev == dev; + + return false; +} + +/* Assign a cache tag with specified type to domain. */ +static int cache_tag_assign(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid, + enum cache_tag_type type) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + struct cache_tag *tag, *temp; + unsigned long flags; + + tag = kzalloc(sizeof(*tag), GFP_KERNEL); + if (!tag) + return -ENOMEM; + + tag->type = type; + tag->iommu = iommu; + tag->domain_id = did; + tag->pasid = pasid; + tag->users = 1; + + if (type == CACHE_TAG_DEVTLB || type == CACHE_TAG_NESTING_DEVTLB) + tag->dev = dev; + else + tag->dev = iommu->iommu.dev; + + spin_lock_irqsave(&domain->cache_lock, flags); + list_for_each_entry(temp, &domain->cache_tags, node) { + if (cache_tage_match(temp, did, iommu, dev, pasid, type)) { + temp->users++; + spin_unlock_irqrestore(&domain->cache_lock, flags); + kfree(tag); + trace_cache_tag_assign(temp); + return 0; + } + } + list_add_tail(&tag->node, &domain->cache_tags); + spin_unlock_irqrestore(&domain->cache_lock, flags); + trace_cache_tag_assign(tag); + + return 0; +} + +/* Unassign a cache tag with specified type from domain. */ +static void cache_tag_unassign(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid, + enum cache_tag_type type) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + struct cache_tag *tag; + unsigned long flags; + + spin_lock_irqsave(&domain->cache_lock, flags); + list_for_each_entry(tag, &domain->cache_tags, node) { + if (cache_tage_match(tag, did, iommu, dev, pasid, type)) { + trace_cache_tag_unassign(tag); + if (--tag->users == 0) { + list_del(&tag->node); + kfree(tag); + } + break; + } + } + spin_unlock_irqrestore(&domain->cache_lock, flags); +} + +static int __cache_tag_assign_domain(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + int ret; + + ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_IOTLB); + if (ret || !info->ats_enabled) + return ret; + + ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_DEVTLB); + if (ret) + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_IOTLB); + + return ret; +} + +static void __cache_tag_unassign_domain(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_IOTLB); + + if (info->ats_enabled) + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_DEVTLB); +} + +static int __cache_tag_assign_parent_domain(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + int ret; + + ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_NESTING_IOTLB); + if (ret || !info->ats_enabled) + return ret; + + ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_NESTING_DEVTLB); + if (ret) + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_NESTING_IOTLB); + + return ret; +} + +static void __cache_tag_unassign_parent_domain(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_NESTING_IOTLB); + + if (info->ats_enabled) + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_NESTING_DEVTLB); +} + +static u16 domain_get_id_for_dev(struct dmar_domain *domain, struct device *dev) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + + /* + * The driver assigns different domain IDs for all domains except + * the SVA type. + */ + if (domain->domain.type == IOMMU_DOMAIN_SVA) + return FLPT_DEFAULT_DID; + + return domain_id_iommu(domain, iommu); +} + +/* + * Assign cache tags to a domain when it's associated with a device's + * PASID using a specific domain ID. + * + * On success (return value of 0), cache tags are created and added to the + * domain's cache tag list. On failure (negative return value), an error + * code is returned indicating the reason for the failure. + */ +int cache_tag_assign_domain(struct dmar_domain *domain, + struct device *dev, ioasid_t pasid) +{ + u16 did = domain_get_id_for_dev(domain, dev); + int ret; + + ret = __cache_tag_assign_domain(domain, did, dev, pasid); + if (ret || domain->domain.type != IOMMU_DOMAIN_NESTED) + return ret; + + ret = __cache_tag_assign_parent_domain(domain->s2_domain, did, dev, pasid); + if (ret) + __cache_tag_unassign_domain(domain, did, dev, pasid); + + return ret; +} + +/* + * Remove the cache tags associated with a device's PASID when the domain is + * detached from the device. + * + * The cache tags must be previously assigned to the domain by calling the + * assign interface. + */ +void cache_tag_unassign_domain(struct dmar_domain *domain, + struct device *dev, ioasid_t pasid) +{ + u16 did = domain_get_id_for_dev(domain, dev); + + __cache_tag_unassign_domain(domain, did, dev, pasid); + if (domain->domain.type == IOMMU_DOMAIN_NESTED) + __cache_tag_unassign_parent_domain(domain->s2_domain, did, dev, pasid); +} + +static unsigned long calculate_psi_aligned_address(unsigned long start, + unsigned long end, + unsigned long *_pages, + unsigned long *_mask) +{ + unsigned long pages = aligned_nrpages(start, end - start + 1); + unsigned long aligned_pages = __roundup_pow_of_two(pages); + unsigned long bitmask = aligned_pages - 1; + unsigned long mask = ilog2(aligned_pages); + unsigned long pfn = IOVA_PFN(start); + + /* + * PSI masks the low order bits of the base address. If the + * address isn't aligned to the mask, then compute a mask value + * needed to ensure the target range is flushed. + */ + if (unlikely(bitmask & pfn)) { + unsigned long end_pfn = pfn + pages - 1, shared_bits; + + /* + * Since end_pfn <= pfn + bitmask, the only way bits + * higher than bitmask can differ in pfn and end_pfn is + * by carrying. This means after masking out bitmask, + * high bits starting with the first set bit in + * shared_bits are all equal in both pfn and end_pfn. + */ + shared_bits = ~(pfn ^ end_pfn) & ~bitmask; + mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG; + } + + *_pages = aligned_pages; + *_mask = mask; + + return ALIGN_DOWN(start, VTD_PAGE_SIZE << mask); +} + +/* + * Invalidates a range of IOVA from @start (inclusive) to @end (inclusive) + * when the memory mappings in the target domain have been modified. + */ +void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start, + unsigned long end, int ih) +{ + unsigned long pages, mask, addr; + struct cache_tag *tag; + unsigned long flags; + + addr = calculate_psi_aligned_address(start, end, &pages, &mask); + + spin_lock_irqsave(&domain->cache_lock, flags); + list_for_each_entry(tag, &domain->cache_tags, node) { + struct intel_iommu *iommu = tag->iommu; + struct device_domain_info *info; + u16 sid; + + switch (tag->type) { + case CACHE_TAG_IOTLB: + case CACHE_TAG_NESTING_IOTLB: + if (domain->use_first_level) { + qi_flush_piotlb(iommu, tag->domain_id, + tag->pasid, addr, pages, ih); + } else { + /* + * Fallback to domain selective flush if no + * PSI support or the size is too big. + */ + if (!cap_pgsel_inv(iommu->cap) || + mask > cap_max_amask_val(iommu->cap)) + iommu->flush.flush_iotlb(iommu, tag->domain_id, + 0, 0, DMA_TLB_DSI_FLUSH); + else + iommu->flush.flush_iotlb(iommu, tag->domain_id, + addr | ih, mask, + DMA_TLB_PSI_FLUSH); + } + break; + case CACHE_TAG_NESTING_DEVTLB: + /* + * Address translation cache in device side caches the + * result of nested translation. There is no easy way + * to identify the exact set of nested translations + * affected by a change in S2. So just flush the entire + * device cache. + */ + addr = 0; + mask = MAX_AGAW_PFN_WIDTH; + fallthrough; + case CACHE_TAG_DEVTLB: + info = dev_iommu_priv_get(tag->dev); + sid = PCI_DEVID(info->bus, info->devfn); + + if (tag->pasid == IOMMU_NO_PASID) + qi_flush_dev_iotlb(iommu, sid, info->pfsid, + info->ats_qdep, addr, mask); + else + qi_flush_dev_iotlb_pasid(iommu, sid, info->pfsid, + tag->pasid, info->ats_qdep, + addr, mask); + + quirk_extra_dev_tlb_flush(info, addr, mask, tag->pasid, info->ats_qdep); + break; + } + + trace_cache_tag_flush_range(tag, start, end, addr, pages, mask); + } + spin_unlock_irqrestore(&domain->cache_lock, flags); +} + +/* + * Invalidates all ranges of IOVA when the memory mappings in the target + * domain have been modified. + */ +void cache_tag_flush_all(struct dmar_domain *domain) +{ + struct cache_tag *tag; + unsigned long flags; + + spin_lock_irqsave(&domain->cache_lock, flags); + list_for_each_entry(tag, &domain->cache_tags, node) { + struct intel_iommu *iommu = tag->iommu; + struct device_domain_info *info; + u16 sid; + + switch (tag->type) { + case CACHE_TAG_IOTLB: + case CACHE_TAG_NESTING_IOTLB: + if (domain->use_first_level) + qi_flush_piotlb(iommu, tag->domain_id, + tag->pasid, 0, -1, 0); + else + iommu->flush.flush_iotlb(iommu, tag->domain_id, + 0, 0, DMA_TLB_DSI_FLUSH); + break; + case CACHE_TAG_DEVTLB: + case CACHE_TAG_NESTING_DEVTLB: + info = dev_iommu_priv_get(tag->dev); + sid = PCI_DEVID(info->bus, info->devfn); + + qi_flush_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep, + 0, MAX_AGAW_PFN_WIDTH); + quirk_extra_dev_tlb_flush(info, 0, MAX_AGAW_PFN_WIDTH, + IOMMU_NO_PASID, info->ats_qdep); + break; + } + + trace_cache_tag_flush_all(tag); + } + spin_unlock_irqrestore(&domain->cache_lock, flags); +} + +/* + * Invalidate a range of IOVA when new mappings are created in the target + * domain. + * + * - VT-d spec, Section 6.1 Caching Mode: When the CM field is reported as + * Set, any software updates to remapping structures other than first- + * stage mapping requires explicit invalidation of the caches. + * - VT-d spec, Section 6.8 Write Buffer Flushing: For hardware that requires + * write buffer flushing, software must explicitly perform write-buffer + * flushing, if cache invalidation is not required. + */ +void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, + unsigned long end) +{ + unsigned long pages, mask, addr; + struct cache_tag *tag; + unsigned long flags; + + addr = calculate_psi_aligned_address(start, end, &pages, &mask); + + spin_lock_irqsave(&domain->cache_lock, flags); + list_for_each_entry(tag, &domain->cache_tags, node) { + struct intel_iommu *iommu = tag->iommu; + + if (!cap_caching_mode(iommu->cap) || domain->use_first_level) { + iommu_flush_write_buffer(iommu); + continue; + } + + if (tag->type == CACHE_TAG_IOTLB || + tag->type == CACHE_TAG_NESTING_IOTLB) { + /* + * Fallback to domain selective flush if no + * PSI support or the size is too big. + */ + if (!cap_pgsel_inv(iommu->cap) || + mask > cap_max_amask_val(iommu->cap)) + iommu->flush.flush_iotlb(iommu, tag->domain_id, + 0, 0, DMA_TLB_DSI_FLUSH); + else + iommu->flush.flush_iotlb(iommu, tag->domain_id, + addr, mask, + DMA_TLB_PSI_FLUSH); + } + + trace_cache_tag_flush_range_np(tag, start, end, addr, pages, mask); + } + spin_unlock_irqrestore(&domain->cache_lock, flags); +} diff --git a/drivers/iommu/intel/debugfs.c b/drivers/iommu/intel/debugfs.c index 86b506af7daa..affbf4a1558d 100644 --- a/drivers/iommu/intel/debugfs.c +++ b/drivers/iommu/intel/debugfs.c @@ -706,7 +706,6 @@ static ssize_t dmar_perf_latency_write(struct file *filp, dmar_latency_disable(iommu, DMAR_LATENCY_INV_IOTLB); dmar_latency_disable(iommu, DMAR_LATENCY_INV_DEVTLB); dmar_latency_disable(iommu, DMAR_LATENCY_INV_IEC); - dmar_latency_disable(iommu, DMAR_LATENCY_PRQ); } rcu_read_unlock(); break; @@ -728,12 +727,6 @@ static ssize_t dmar_perf_latency_write(struct file *filp, dmar_latency_enable(iommu, DMAR_LATENCY_INV_IEC); rcu_read_unlock(); break; - case 4: - rcu_read_lock(); - for_each_active_iommu(iommu, drhd) - dmar_latency_enable(iommu, DMAR_LATENCY_PRQ); - rcu_read_unlock(); - break; default: return -EINVAL; } diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 36d7427b1202..304e84949ca7 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -32,6 +32,7 @@ #include "iommu.h" #include "../irq_remapping.h" +#include "../iommu-pages.h" #include "perf.h" #include "trace.h" #include "perfmon.h" @@ -1067,7 +1068,6 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) goto error_free_seq_id; } - err = -EINVAL; if (!cap_sagaw(iommu->cap) && (!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) { pr_info("%s: No supported address widths. Not attempting DMA translation.\n", @@ -1187,7 +1187,7 @@ static void free_iommu(struct intel_iommu *iommu) } if (iommu->qi) { - free_page((unsigned long)iommu->qi->desc); + iommu_free_page(iommu->qi->desc); kfree(iommu->qi->desc_status); kfree(iommu->qi); } @@ -1755,7 +1755,8 @@ static void __dmar_enable_qi(struct intel_iommu *iommu) int dmar_enable_qi(struct intel_iommu *iommu) { struct q_inval *qi; - struct page *desc_page; + void *desc; + int order; if (!ecap_qis(iommu->ecap)) return -ENOENT; @@ -1776,19 +1777,19 @@ int dmar_enable_qi(struct intel_iommu *iommu) * Need two pages to accommodate 256 descriptors of 256 bits each * if the remapping hardware supports scalable mode translation. */ - desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, - !!ecap_smts(iommu->ecap)); - if (!desc_page) { + order = ecap_smts(iommu->ecap) ? 1 : 0; + desc = iommu_alloc_pages_node(iommu->node, GFP_ATOMIC, order); + if (!desc) { kfree(qi); iommu->qi = NULL; return -ENOMEM; } - qi->desc = page_address(desc_page); + qi->desc = desc; qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC); if (!qi->desc_status) { - free_page((unsigned long) qi->desc); + iommu_free_page(qi->desc); kfree(qi); iommu->qi = NULL; return -ENOMEM; @@ -2122,7 +2123,7 @@ int dmar_set_interrupt(struct intel_iommu *iommu) return ret; } -int __init enable_drhd_fault_handling(void) +int enable_drhd_fault_handling(unsigned int cpu) { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu; @@ -2132,7 +2133,12 @@ int __init enable_drhd_fault_handling(void) */ for_each_iommu(iommu, drhd) { u32 fault_status; - int ret = dmar_set_interrupt(iommu); + int ret; + + if (iommu->irq || iommu->node != cpu_to_node(cpu)) + continue; + + ret = dmar_set_interrupt(iommu); if (ret) { pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n", diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index a7ecd90303dc..2e9811bf2a4e 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -27,6 +27,7 @@ #include "iommu.h" #include "../dma-iommu.h" #include "../irq_remapping.h" +#include "../iommu-pages.h" #include "pasid.h" #include "cap_audit.h" #include "perfmon.h" @@ -54,11 +55,6 @@ __DOMAIN_MAX_PFN(gaw), (unsigned long)-1)) #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT) -/* IO virtual address start page frame number */ -#define IOVA_START_PFN (1) - -#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) - static void __init check_tylersburg_isoch(void); static int rwbf_quirk; @@ -221,12 +217,11 @@ int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON); int intel_iommu_enabled = 0; EXPORT_SYMBOL_GPL(intel_iommu_enabled); -static int dmar_map_gfx = 1; static int intel_iommu_superpage = 1; static int iommu_identity_mapping; static int iommu_skip_te_disable; +static int disable_igfx_iommu; -#define IDENTMAP_GFX 2 #define IDENTMAP_AZALIA 4 const struct iommu_ops intel_iommu_ops; @@ -265,7 +260,7 @@ static int __init intel_iommu_setup(char *str) no_platform_optin = 1; pr_info("IOMMU disabled\n"); } else if (!strncmp(str, "igfx_off", 8)) { - dmar_map_gfx = 0; + disable_igfx_iommu = 1; pr_info("Disable GFX device mapping\n"); } else if (!strncmp(str, "forcedac", 8)) { pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n"); @@ -298,22 +293,6 @@ static int __init intel_iommu_setup(char *str) } __setup("intel_iommu=", intel_iommu_setup); -void *alloc_pgtable_page(int node, gfp_t gfp) -{ - struct page *page; - void *vaddr = NULL; - - page = alloc_pages_node(node, gfp | __GFP_ZERO, 0); - if (page) - vaddr = page_address(page); - return vaddr; -} - -void free_pgtable_page(void *vaddr) -{ - free_page((unsigned long)vaddr); -} - static int domain_type_is_si(struct dmar_domain *domain) { return domain->domain.type == IOMMU_DOMAIN_IDENTITY; @@ -545,7 +524,7 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, if (!alloc) return NULL; - context = alloc_pgtable_page(iommu->node, GFP_ATOMIC); + context = iommu_alloc_page_node(iommu->node, GFP_ATOMIC); if (!context) return NULL; @@ -719,17 +698,17 @@ static void free_context_table(struct intel_iommu *iommu) for (i = 0; i < ROOT_ENTRY_NR; i++) { context = iommu_context_addr(iommu, i, 0, 0); if (context) - free_pgtable_page(context); + iommu_free_page(context); if (!sm_supported(iommu)) continue; context = iommu_context_addr(iommu, i, 0x80, 0); if (context) - free_pgtable_page(context); + iommu_free_page(context); } - free_pgtable_page(iommu->root_entry); + iommu_free_page(iommu->root_entry); iommu->root_entry = NULL; } @@ -865,9 +844,9 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, break; if (!dma_pte_present(pte)) { - uint64_t pteval; + uint64_t pteval, tmp; - tmp_page = alloc_pgtable_page(domain->nid, gfp); + tmp_page = iommu_alloc_page_node(domain->nid, gfp); if (!tmp_page) return NULL; @@ -877,9 +856,10 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, if (domain->use_first_level) pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS; - if (cmpxchg64(&pte->val, 0ULL, pteval)) + tmp = 0ULL; + if (!try_cmpxchg64(&pte->val, &tmp, pteval)) /* Someone else set it while we were thinking; use theirs. */ - free_pgtable_page(tmp_page); + iommu_free_page(tmp_page); else domain_flush_cache(domain, pte, sizeof(*pte)); } @@ -992,7 +972,7 @@ static void dma_pte_free_level(struct dmar_domain *domain, int level, last_pfn < level_pfn + level_size(level) - 1)) { dma_clear_pte(pte); domain_flush_cache(domain, pte, sizeof(*pte)); - free_pgtable_page(level_pte); + iommu_free_page(level_pte); } next: pfn += level_size(level); @@ -1016,7 +996,7 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain, /* free pgd */ if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) { - free_pgtable_page(domain->pgd); + iommu_free_page(domain->pgd); domain->pgd = NULL; } } @@ -1118,7 +1098,7 @@ static int iommu_alloc_root_entry(struct intel_iommu *iommu) { struct root_entry *root; - root = alloc_pgtable_page(iommu->node, GFP_ATOMIC); + root = iommu_alloc_page_node(iommu->node, GFP_ATOMIC); if (!root) { pr_err("Allocating root entry for %s failed\n", iommu->name); @@ -1394,197 +1374,9 @@ static void __iommu_flush_dev_iotlb(struct device_domain_info *info, quirk_extra_dev_tlb_flush(info, addr, mask, IOMMU_NO_PASID, qdep); } -static void iommu_flush_dev_iotlb(struct dmar_domain *domain, - u64 addr, unsigned mask) -{ - struct dev_pasid_info *dev_pasid; - struct device_domain_info *info; - unsigned long flags; - - if (!domain->has_iotlb_device) - return; - - spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry(info, &domain->devices, link) - __iommu_flush_dev_iotlb(info, addr, mask); - - list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) { - info = dev_iommu_priv_get(dev_pasid->dev); - - if (!info->ats_enabled) - continue; - - qi_flush_dev_iotlb_pasid(info->iommu, - PCI_DEVID(info->bus, info->devfn), - info->pfsid, dev_pasid->pasid, - info->ats_qdep, addr, - mask); - } - spin_unlock_irqrestore(&domain->lock, flags); -} - -static void domain_flush_pasid_iotlb(struct intel_iommu *iommu, - struct dmar_domain *domain, u64 addr, - unsigned long npages, bool ih) -{ - u16 did = domain_id_iommu(domain, iommu); - struct dev_pasid_info *dev_pasid; - unsigned long flags; - - spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) - qi_flush_piotlb(iommu, did, dev_pasid->pasid, addr, npages, ih); - - if (!list_empty(&domain->devices)) - qi_flush_piotlb(iommu, did, IOMMU_NO_PASID, addr, npages, ih); - spin_unlock_irqrestore(&domain->lock, flags); -} - -static void __iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did, - unsigned long pfn, unsigned int pages, - int ih) -{ - unsigned int aligned_pages = __roundup_pow_of_two(pages); - unsigned long bitmask = aligned_pages - 1; - unsigned int mask = ilog2(aligned_pages); - u64 addr = (u64)pfn << VTD_PAGE_SHIFT; - - /* - * PSI masks the low order bits of the base address. If the - * address isn't aligned to the mask, then compute a mask value - * needed to ensure the target range is flushed. - */ - if (unlikely(bitmask & pfn)) { - unsigned long end_pfn = pfn + pages - 1, shared_bits; - - /* - * Since end_pfn <= pfn + bitmask, the only way bits - * higher than bitmask can differ in pfn and end_pfn is - * by carrying. This means after masking out bitmask, - * high bits starting with the first set bit in - * shared_bits are all equal in both pfn and end_pfn. - */ - shared_bits = ~(pfn ^ end_pfn) & ~bitmask; - mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG; - } - - /* - * Fallback to domain selective flush if no PSI support or - * the size is too big. - */ - if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap)) - iommu->flush.flush_iotlb(iommu, did, 0, 0, - DMA_TLB_DSI_FLUSH); - else - iommu->flush.flush_iotlb(iommu, did, addr | ih, mask, - DMA_TLB_PSI_FLUSH); -} - -static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, - struct dmar_domain *domain, - unsigned long pfn, unsigned int pages, - int ih, int map) -{ - unsigned int aligned_pages = __roundup_pow_of_two(pages); - unsigned int mask = ilog2(aligned_pages); - uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT; - u16 did = domain_id_iommu(domain, iommu); - - if (WARN_ON(!pages)) - return; - - if (ih) - ih = 1 << 6; - - if (domain->use_first_level) - domain_flush_pasid_iotlb(iommu, domain, addr, pages, ih); - else - __iommu_flush_iotlb_psi(iommu, did, pfn, pages, ih); - - /* - * In caching mode, changes of pages from non-present to present require - * flush. However, device IOTLB doesn't need to be flushed in this case. - */ - if (!cap_caching_mode(iommu->cap) || !map) - iommu_flush_dev_iotlb(domain, addr, mask); -} - -/* Notification for newly created mappings */ -static void __mapping_notify_one(struct intel_iommu *iommu, struct dmar_domain *domain, - unsigned long pfn, unsigned int pages) -{ - /* - * It's a non-present to present mapping. Only flush if caching mode - * and second level. - */ - if (cap_caching_mode(iommu->cap) && !domain->use_first_level) - iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1); - else - iommu_flush_write_buffer(iommu); -} - -/* - * Flush the relevant caches in nested translation if the domain - * also serves as a parent - */ -static void parent_domain_flush(struct dmar_domain *domain, - unsigned long pfn, - unsigned long pages, int ih) -{ - struct dmar_domain *s1_domain; - - spin_lock(&domain->s1_lock); - list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) { - struct device_domain_info *device_info; - struct iommu_domain_info *info; - unsigned long flags; - unsigned long i; - - xa_for_each(&s1_domain->iommu_array, i, info) - __iommu_flush_iotlb_psi(info->iommu, info->did, - pfn, pages, ih); - - if (!s1_domain->has_iotlb_device) - continue; - - spin_lock_irqsave(&s1_domain->lock, flags); - list_for_each_entry(device_info, &s1_domain->devices, link) - /* - * Address translation cache in device side caches the - * result of nested translation. There is no easy way - * to identify the exact set of nested translations - * affected by a change in S2. So just flush the entire - * device cache. - */ - __iommu_flush_dev_iotlb(device_info, 0, - MAX_AGAW_PFN_WIDTH); - spin_unlock_irqrestore(&s1_domain->lock, flags); - } - spin_unlock(&domain->s1_lock); -} - static void intel_flush_iotlb_all(struct iommu_domain *domain) { - struct dmar_domain *dmar_domain = to_dmar_domain(domain); - struct iommu_domain_info *info; - unsigned long idx; - - xa_for_each(&dmar_domain->iommu_array, idx, info) { - struct intel_iommu *iommu = info->iommu; - u16 did = domain_id_iommu(dmar_domain, iommu); - - if (dmar_domain->use_first_level) - domain_flush_pasid_iotlb(iommu, dmar_domain, 0, -1, 0); - else - iommu->flush.flush_iotlb(iommu, did, 0, 0, - DMA_TLB_DSI_FLUSH); - - if (!cap_caching_mode(iommu->cap)) - iommu_flush_dev_iotlb(dmar_domain, 0, MAX_AGAW_PFN_WIDTH); - } - - if (dmar_domain->nested_parent) - parent_domain_flush(dmar_domain, 0, -1, 0); + cache_tag_flush_all(to_dmar_domain(domain)); } static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) @@ -1750,7 +1542,9 @@ static struct dmar_domain *alloc_domain(unsigned int type) domain->has_iotlb_device = false; INIT_LIST_HEAD(&domain->devices); INIT_LIST_HEAD(&domain->dev_pasids); + INIT_LIST_HEAD(&domain->cache_tags); spin_lock_init(&domain->lock); + spin_lock_init(&domain->cache_lock); xa_init(&domain->iommu_array); return domain; @@ -1762,6 +1556,9 @@ int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu) unsigned long ndomains; int num, ret = -ENOSPC; + if (domain->domain.type == IOMMU_DOMAIN_SVA) + return 0; + info = kzalloc(sizeof(*info), GFP_KERNEL); if (!info) return -ENOMEM; @@ -1809,6 +1606,9 @@ void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu) { struct iommu_domain_info *info; + if (domain->domain.type == IOMMU_DOMAIN_SVA) + return; + spin_lock(&iommu->lock); info = xa_load(&domain->iommu_array, iommu->seq_id); if (--info->refcnt == 0) { @@ -1841,7 +1641,7 @@ static void domain_exit(struct dmar_domain *domain) LIST_HEAD(freelist); domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist); - put_pages_list(&freelist); + iommu_put_pages_list(&freelist); } if (WARN_ON(!list_empty(&domain->devices))) @@ -1988,13 +1788,6 @@ domain_context_mapping(struct dmar_domain *domain, struct device *dev) domain_context_mapping_cb, domain); } -/* Returns a number of VTD pages, but aligned to MM page size */ -static unsigned long aligned_nrpages(unsigned long host_addr, size_t size) -{ - host_addr &= ~PAGE_MASK; - return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT; -} - /* Return largest possible superpage level for a given mapping */ static int hardware_largepage_caps(struct dmar_domain *domain, unsigned long iov_pfn, unsigned long phy_pfn, unsigned long pages) @@ -2031,9 +1824,7 @@ static void switch_to_super_page(struct dmar_domain *domain, unsigned long end_pfn, int level) { unsigned long lvl_pages = lvl_to_nr_pages(level); - struct iommu_domain_info *info; struct dma_pte *pte = NULL; - unsigned long i; while (start_pfn <= end_pfn) { if (!pte) @@ -2045,13 +1836,8 @@ static void switch_to_super_page(struct dmar_domain *domain, start_pfn + lvl_pages - 1, level + 1); - xa_for_each(&domain->iommu_array, i, info) - iommu_flush_iotlb_psi(info->iommu, domain, - start_pfn, lvl_pages, - 0, 0); - if (domain->nested_parent) - parent_domain_flush(domain, start_pfn, - lvl_pages, 0); + cache_tag_flush_range(domain, start_pfn << VTD_PAGE_SHIFT, + end_pfn << VTD_PAGE_SHIFT, 0); } pte++; @@ -2128,8 +1914,8 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, /* We don't need lock here, nobody else * touches the iova range */ - tmp = cmpxchg64_local(&pte->val, 0ULL, pteval); - if (tmp) { + tmp = 0ULL; + if (!try_cmpxchg64_local(&pte->val, &tmp, pteval)) { static int dumps = 5; pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n", iov_pfn, tmp, (unsigned long long)pteval); @@ -2327,6 +2113,13 @@ static int dmar_domain_attach_device(struct dmar_domain *domain, ret = domain_attach_iommu(domain, iommu); if (ret) return ret; + + ret = cache_tag_assign_domain(domain, dev, IOMMU_NO_PASID); + if (ret) { + domain_detach_iommu(domain, iommu); + return ret; + } + info->domain = domain; spin_lock_irqsave(&domain->lock, flags); list_add(&info->link, &domain->devices); @@ -2402,9 +2195,6 @@ static int device_def_domain_type(struct device *dev) if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev)) return IOMMU_DOMAIN_IDENTITY; - - if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev)) - return IOMMU_DOMAIN_IDENTITY; } return 0; @@ -2497,7 +2287,7 @@ static int copy_context_table(struct intel_iommu *iommu, if (!old_ce) goto out; - new_ce = alloc_pgtable_page(iommu->node, GFP_KERNEL); + new_ce = iommu_alloc_page_node(iommu->node, GFP_KERNEL); if (!new_ce) goto out_unmap; @@ -2705,9 +2495,6 @@ static int __init init_dmars(void) iommu_set_root_entry(iommu); } - if (!dmar_map_gfx) - iommu_identity_mapping |= IDENTMAP_GFX; - check_tylersburg_isoch(); ret = si_domain_init(hw_pass_through); @@ -2798,7 +2585,7 @@ static void __init init_no_remapping_devices(void) /* This IOMMU has *only* gfx devices. Either bypass it or set the gfx_mapped flag, as appropriate */ drhd->gfx_dedicated = 1; - if (!dmar_map_gfx) + if (disable_igfx_iommu) drhd->ignored = 1; } } @@ -3414,19 +3201,10 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, case MEM_OFFLINE: case MEM_CANCEL_ONLINE: { - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; LIST_HEAD(freelist); domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist); - - rcu_read_lock(); - for_each_active_iommu(iommu, drhd) - iommu_flush_iotlb_psi(iommu, si_domain, - start_vpfn, mhp->nr_pages, - list_empty(&freelist), 0); - rcu_read_unlock(); - put_pages_list(&freelist); + iommu_put_pages_list(&freelist); } break; } @@ -3815,6 +3593,7 @@ void device_block_translation(struct device *dev) list_del(&info->link); spin_unlock_irqrestore(&info->domain->lock, flags); + cache_tag_unassign_domain(info->domain, dev, IOMMU_NO_PASID); domain_detach_iommu(info->domain, iommu); info->domain = NULL; } @@ -3833,7 +3612,7 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width) domain->max_addr = 0; /* always allocate the top pgd */ - domain->pgd = alloc_pgtable_page(domain->nid, GFP_ATOMIC); + domain->pgd = iommu_alloc_page_node(domain->nid, GFP_ATOMIC); if (!domain->pgd) return -ENOMEM; domain_flush_cache(domain, domain->pgd, PAGE_SIZE); @@ -3882,8 +3661,6 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) return domain; case IOMMU_DOMAIN_IDENTITY: return &si_domain->domain; - case IOMMU_DOMAIN_SVA: - return intel_svm_domain_alloc(); default: return NULL; } @@ -3987,7 +3764,7 @@ int prepare_domain_attach_device(struct iommu_domain *domain, pte = dmar_domain->pgd; if (dma_pte_present(pte)) { dmar_domain->pgd = phys_to_virt(dma_pte_addr(pte)); - free_pgtable_page(pte); + iommu_free_page(pte); } dmar_domain->agaw--; } @@ -4122,26 +3899,9 @@ static size_t intel_iommu_unmap_pages(struct iommu_domain *domain, static void intel_iommu_tlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { - struct dmar_domain *dmar_domain = to_dmar_domain(domain); - unsigned long iova_pfn = IOVA_PFN(gather->start); - size_t size = gather->end - gather->start; - struct iommu_domain_info *info; - unsigned long start_pfn; - unsigned long nrpages; - unsigned long i; - - nrpages = aligned_nrpages(gather->start, size); - start_pfn = mm_to_dma_pfn_start(iova_pfn); - - xa_for_each(&dmar_domain->iommu_array, i, info) - iommu_flush_iotlb_psi(info->iommu, dmar_domain, - start_pfn, nrpages, - list_empty(&gather->freelist), 0); - - if (dmar_domain->nested_parent) - parent_domain_flush(dmar_domain, start_pfn, nrpages, - list_empty(&gather->freelist)); - put_pages_list(&gather->freelist); + cache_tag_flush_range(to_dmar_domain(domain), gather->start, + gather->end, list_empty(&gather->freelist)); + iommu_put_pages_list(&gather->freelist); } static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, @@ -4352,12 +4112,6 @@ static void intel_iommu_release_device(struct device *dev) set_dma_ops(dev, NULL); } -static void intel_iommu_probe_finalize(struct device *dev) -{ - set_dma_ops(dev, NULL); - iommu_setup_dma_ops(dev, 0, U64_MAX); -} - static void intel_iommu_get_resv_regions(struct device *device, struct list_head *head) { @@ -4579,41 +4333,20 @@ static bool risky_device(struct pci_dev *pdev) static int intel_iommu_iotlb_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) { - struct dmar_domain *dmar_domain = to_dmar_domain(domain); - unsigned long pages = aligned_nrpages(iova, size); - unsigned long pfn = iova >> VTD_PAGE_SHIFT; - struct iommu_domain_info *info; - unsigned long i; + cache_tag_flush_range_np(to_dmar_domain(domain), iova, iova + size - 1); - xa_for_each(&dmar_domain->iommu_array, i, info) - __mapping_notify_one(info->iommu, dmar_domain, pfn, pages); return 0; } -static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) +static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid, + struct iommu_domain *domain) { struct device_domain_info *info = dev_iommu_priv_get(dev); + struct dmar_domain *dmar_domain = to_dmar_domain(domain); struct dev_pasid_info *curr, *dev_pasid = NULL; struct intel_iommu *iommu = info->iommu; - struct dmar_domain *dmar_domain; - struct iommu_domain *domain; unsigned long flags; - domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0); - if (WARN_ON_ONCE(!domain)) - goto out_tear_down; - - /* - * The SVA implementation needs to handle its own stuffs like the mm - * notification. Before consolidating that code into iommu core, let - * the intel sva code handle it. - */ - if (domain->type == IOMMU_DOMAIN_SVA) { - intel_svm_remove_dev_pasid(dev, pasid); - goto out_tear_down; - } - - dmar_domain = to_dmar_domain(domain); spin_lock_irqsave(&dmar_domain->lock, flags); list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) { if (curr->dev == dev && curr->pasid == pasid) { @@ -4625,10 +4358,10 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) WARN_ON_ONCE(!dev_pasid); spin_unlock_irqrestore(&dmar_domain->lock, flags); + cache_tag_unassign_domain(dmar_domain, dev, pasid); domain_detach_iommu(dmar_domain, iommu); intel_iommu_debugfs_remove_dev_pasid(dev_pasid); kfree(dev_pasid); -out_tear_down: intel_pasid_tear_down_entry(iommu, dev, pasid, false); intel_drain_pasid_prq(dev, pasid); } @@ -4664,6 +4397,10 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain, if (ret) goto out_free; + ret = cache_tag_assign_domain(dmar_domain, dev, pasid); + if (ret) + goto out_detach_iommu; + if (domain_type_is_si(dmar_domain)) ret = intel_pasid_setup_pass_through(iommu, dev, pasid); else if (dmar_domain->use_first_level) @@ -4673,7 +4410,7 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain, ret = intel_pasid_setup_second_level(iommu, dmar_domain, dev, pasid); if (ret) - goto out_detach_iommu; + goto out_unassign_tag; dev_pasid->dev = dev; dev_pasid->pasid = pasid; @@ -4685,6 +4422,8 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain, intel_iommu_debugfs_create_dev_pasid(dev_pasid); return 0; +out_unassign_tag: + cache_tag_unassign_domain(dmar_domain, dev, pasid); out_detach_iommu: domain_detach_iommu(dmar_domain, iommu); out_free: @@ -4841,8 +4580,8 @@ const struct iommu_ops intel_iommu_ops = { .hw_info = intel_iommu_hw_info, .domain_alloc = intel_iommu_domain_alloc, .domain_alloc_user = intel_iommu_domain_alloc_user, + .domain_alloc_sva = intel_svm_domain_alloc, .probe_device = intel_iommu_probe_device, - .probe_finalize = intel_iommu_probe_finalize, .release_device = intel_iommu_release_device, .get_resv_regions = intel_iommu_get_resv_regions, .device_group = intel_iommu_device_group, @@ -4875,7 +4614,7 @@ static void quirk_iommu_igfx(struct pci_dev *dev) return; pci_info(dev, "Disabling IOMMU for graphics on this chipset\n"); - dmar_map_gfx = 0; + disable_igfx_iommu = 1; } /* G4x/GM45 integrated gfx dmar support is totally busted. */ @@ -4956,8 +4695,8 @@ static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev) if (!(ggc & GGC_MEMORY_VT_ENABLED)) { pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n"); - dmar_map_gfx = 0; - } else if (dmar_map_gfx) { + disable_igfx_iommu = 1; + } else if (!disable_igfx_iommu) { /* we have to ensure the gfx device is idle before we flush */ pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n"); iommu_set_dma_strict(); diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 404d2476a877..eaf015b4353b 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -35,6 +35,8 @@ #define VTD_PAGE_MASK (((u64)-1) << VTD_PAGE_SHIFT) #define VTD_PAGE_ALIGN(addr) (((addr) + VTD_PAGE_SIZE - 1) & VTD_PAGE_MASK) +#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) + #define VTD_STRIDE_SHIFT (9) #define VTD_STRIDE_MASK (((u64)-1) << VTD_STRIDE_SHIFT) @@ -455,7 +457,6 @@ enum { /* Page group response descriptor QW0 */ #define QI_PGRP_PASID_P(p) (((u64)(p)) << 4) -#define QI_PGRP_PDP(p) (((u64)(p)) << 5) #define QI_PGRP_RESP_CODE(res) (((u64)(res)) << 12) #define QI_PGRP_DID(rid) (((u64)(rid)) << 16) #define QI_PGRP_PASID(pasid) (((u64)(pasid)) << 32) @@ -607,6 +608,9 @@ struct dmar_domain { struct list_head devices; /* all devices' list */ struct list_head dev_pasids; /* all attached pasids */ + spinlock_t cache_lock; /* Protect the cache tag list */ + struct list_head cache_tags; /* Cache tag list */ + int iommu_superpage;/* Level of superpages supported: 0 == 4KiB (no superpages), 1 == 2MiB, 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */ @@ -644,6 +648,11 @@ struct dmar_domain { /* link to parent domain siblings */ struct list_head s2_link; }; + + /* SVA domain */ + struct { + struct mmu_notifier notifier; + }; }; struct iommu_domain domain; /* generic domain data structure for @@ -1038,6 +1047,19 @@ static inline void context_set_sm_pre(struct context_entry *context) context->lo |= BIT_ULL(4); } +/* Returns a number of VTD pages, but aligned to MM page size */ +static inline unsigned long aligned_nrpages(unsigned long host_addr, size_t size) +{ + host_addr &= ~PAGE_MASK; + return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT; +} + +/* Return a size from number of VTD pages. */ +static inline unsigned long nrpages_to_size(unsigned long npages) +{ + return npages << VTD_PAGE_SHIFT; +} + /* Convert value to context PASID directory size field coding. */ #define context_pdts(pds) (((pds) & 0x7) << 9) @@ -1085,48 +1107,60 @@ void domain_update_iommu_cap(struct dmar_domain *domain); int dmar_ir_support(void); -void *alloc_pgtable_page(int node, gfp_t gfp); -void free_pgtable_page(void *vaddr); void iommu_flush_write_buffer(struct intel_iommu *iommu); struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, const struct iommu_user_data *user_data); struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid); +enum cache_tag_type { + CACHE_TAG_IOTLB, + CACHE_TAG_DEVTLB, + CACHE_TAG_NESTING_IOTLB, + CACHE_TAG_NESTING_DEVTLB, +}; + +struct cache_tag { + struct list_head node; + enum cache_tag_type type; + struct intel_iommu *iommu; + /* + * The @dev field represents the location of the cache. For IOTLB, it + * resides on the IOMMU hardware. @dev stores the device pointer to + * the IOMMU hardware. For DevTLB, it locates in the PCIe endpoint. + * @dev stores the device pointer to that endpoint. + */ + struct device *dev; + u16 domain_id; + ioasid_t pasid; + unsigned int users; +}; + +int cache_tag_assign_domain(struct dmar_domain *domain, + struct device *dev, ioasid_t pasid); +void cache_tag_unassign_domain(struct dmar_domain *domain, + struct device *dev, ioasid_t pasid); +void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start, + unsigned long end, int ih); +void cache_tag_flush_all(struct dmar_domain *domain); +void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, + unsigned long end); + #ifdef CONFIG_INTEL_IOMMU_SVM void intel_svm_check(struct intel_iommu *iommu); int intel_svm_enable_prq(struct intel_iommu *iommu); int intel_svm_finish_prq(struct intel_iommu *iommu); void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, struct iommu_page_response *msg); -struct iommu_domain *intel_svm_domain_alloc(void); -void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid); +struct iommu_domain *intel_svm_domain_alloc(struct device *dev, + struct mm_struct *mm); void intel_drain_pasid_prq(struct device *dev, u32 pasid); - -struct intel_svm_dev { - struct list_head list; - struct rcu_head rcu; - struct device *dev; - struct intel_iommu *iommu; - u16 did; - u16 sid, qdep; -}; - -struct intel_svm { - struct mmu_notifier notifier; - struct mm_struct *mm; - u32 pasid; - struct list_head devs; -}; #else static inline void intel_svm_check(struct intel_iommu *iommu) {} static inline void intel_drain_pasid_prq(struct device *dev, u32 pasid) {} -static inline struct iommu_domain *intel_svm_domain_alloc(void) -{ - return NULL; -} - -static inline void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid) +static inline struct iommu_domain *intel_svm_domain_alloc(struct device *dev, + struct mm_struct *mm) { + return ERR_PTR(-ENODEV); } #endif diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 712ebfc9870c..e4a70886678c 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -23,6 +23,7 @@ #include "iommu.h" #include "../irq_remapping.h" +#include "../iommu-pages.h" #include "cap_audit.h" enum irq_mode { @@ -84,7 +85,7 @@ static const struct irq_domain_ops intel_ir_domain_ops; static void iommu_disable_irq_remapping(struct intel_iommu *iommu); static int __init parse_ioapics_under_ir(void); -static const struct msi_parent_ops dmar_msi_parent_ops, virt_dmar_msi_parent_ops; +static const struct msi_parent_ops dmar_msi_parent_ops; static bool ir_pre_enabled(struct intel_iommu *iommu) { @@ -529,7 +530,7 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) struct ir_table *ir_table; struct fwnode_handle *fn; unsigned long *bitmap; - struct page *pages; + void *ir_table_base; if (iommu->ir_table) return 0; @@ -538,9 +539,9 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) if (!ir_table) return -ENOMEM; - pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, - INTR_REMAP_PAGE_ORDER); - if (!pages) { + ir_table_base = iommu_alloc_pages_node(iommu->node, GFP_KERNEL, + INTR_REMAP_PAGE_ORDER); + if (!ir_table_base) { pr_err("IR%d: failed to allocate pages of order %d\n", iommu->seq_id, INTR_REMAP_PAGE_ORDER); goto out_free_table; @@ -569,13 +570,9 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) irq_domain_update_bus_token(iommu->ir_domain, DOMAIN_BUS_DMAR); iommu->ir_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT | IRQ_DOMAIN_FLAG_ISOLATED_MSI; + iommu->ir_domain->msi_parent_ops = &dmar_msi_parent_ops; - if (cap_caching_mode(iommu->cap)) - iommu->ir_domain->msi_parent_ops = &virt_dmar_msi_parent_ops; - else - iommu->ir_domain->msi_parent_ops = &dmar_msi_parent_ops; - - ir_table->base = page_address(pages); + ir_table->base = ir_table_base; ir_table->bitmap = bitmap; iommu->ir_table = ir_table; @@ -624,7 +621,7 @@ out_free_fwnode: out_free_bitmap: bitmap_free(bitmap); out_free_pages: - __free_pages(pages, INTR_REMAP_PAGE_ORDER); + iommu_free_pages(ir_table_base, INTR_REMAP_PAGE_ORDER); out_free_table: kfree(ir_table); @@ -645,8 +642,7 @@ static void intel_teardown_irq_remapping(struct intel_iommu *iommu) irq_domain_free_fwnode(fn); iommu->ir_domain = NULL; } - free_pages((unsigned long)iommu->ir_table->base, - INTR_REMAP_PAGE_ORDER); + iommu_free_pages(iommu->ir_table->base, INTR_REMAP_PAGE_ORDER); bitmap_free(iommu->ir_table->bitmap); kfree(iommu->ir_table); iommu->ir_table = NULL; @@ -1526,20 +1522,11 @@ static const struct irq_domain_ops intel_ir_domain_ops = { }; static const struct msi_parent_ops dmar_msi_parent_ops = { - .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | - MSI_FLAG_MULTI_PCI_MSI | - MSI_FLAG_PCI_IMS, + .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | MSI_FLAG_MULTI_PCI_MSI, .prefix = "IR-", .init_dev_msi_info = msi_parent_init_dev_msi_info, }; -static const struct msi_parent_ops virt_dmar_msi_parent_ops = { - .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | - MSI_FLAG_MULTI_PCI_MSI, - .prefix = "vIR-", - .init_dev_msi_info = msi_parent_init_dev_msi_info, -}; - /* * Support of Interrupt Remapping Unit Hotplug */ diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index a7d68f3d518a..16a2bcf5cfeb 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c @@ -52,13 +52,14 @@ static int intel_nested_attach_dev(struct iommu_domain *domain, return ret; } + ret = cache_tag_assign_domain(dmar_domain, dev, IOMMU_NO_PASID); + if (ret) + goto detach_iommu; + ret = intel_pasid_setup_nested(iommu, dev, IOMMU_NO_PASID, dmar_domain); - if (ret) { - domain_detach_iommu(dmar_domain, iommu); - dev_err_ratelimited(dev, "Failed to setup pasid entry\n"); - return ret; - } + if (ret) + goto unassign_tag; info->domain = dmar_domain; spin_lock_irqsave(&dmar_domain->lock, flags); @@ -68,6 +69,12 @@ static int intel_nested_attach_dev(struct iommu_domain *domain, domain_update_iotlb(dmar_domain); return 0; +unassign_tag: + cache_tag_unassign_domain(dmar_domain, dev, IOMMU_NO_PASID); +detach_iommu: + domain_detach_iommu(dmar_domain, iommu); + + return ret; } static void intel_nested_domain_free(struct iommu_domain *domain) @@ -81,50 +88,6 @@ static void intel_nested_domain_free(struct iommu_domain *domain) kfree(dmar_domain); } -static void nested_flush_dev_iotlb(struct dmar_domain *domain, u64 addr, - unsigned int mask) -{ - struct device_domain_info *info; - unsigned long flags; - u16 sid, qdep; - - spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry(info, &domain->devices, link) { - if (!info->ats_enabled) - continue; - sid = info->bus << 8 | info->devfn; - qdep = info->ats_qdep; - qi_flush_dev_iotlb(info->iommu, sid, info->pfsid, - qdep, addr, mask); - quirk_extra_dev_tlb_flush(info, addr, mask, - IOMMU_NO_PASID, qdep); - } - spin_unlock_irqrestore(&domain->lock, flags); -} - -static void intel_nested_flush_cache(struct dmar_domain *domain, u64 addr, - u64 npages, bool ih) -{ - struct iommu_domain_info *info; - unsigned int mask; - unsigned long i; - - xa_for_each(&domain->iommu_array, i, info) - qi_flush_piotlb(info->iommu, - domain_id_iommu(domain, info->iommu), - IOMMU_NO_PASID, addr, npages, ih); - - if (!domain->has_iotlb_device) - return; - - if (npages == U64_MAX) - mask = 64 - VTD_PAGE_SHIFT; - else - mask = ilog2(__roundup_pow_of_two(npages)); - - nested_flush_dev_iotlb(domain, addr, mask); -} - static int intel_nested_cache_invalidate_user(struct iommu_domain *domain, struct iommu_user_data_array *array) { @@ -157,9 +120,9 @@ static int intel_nested_cache_invalidate_user(struct iommu_domain *domain, break; } - intel_nested_flush_cache(dmar_domain, inv_entry.addr, - inv_entry.npages, - inv_entry.flags & IOMMU_VTD_INV_FLAGS_LEAF); + cache_tag_flush_range(dmar_domain, inv_entry.addr, + inv_entry.addr + nrpages_to_size(inv_entry.npages) - 1, + inv_entry.flags & IOMMU_VTD_INV_FLAGS_LEAF); processed++; } @@ -206,7 +169,9 @@ struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, domain->domain.type = IOMMU_DOMAIN_NESTED; INIT_LIST_HEAD(&domain->devices); INIT_LIST_HEAD(&domain->dev_pasids); + INIT_LIST_HEAD(&domain->cache_tags); spin_lock_init(&domain->lock); + spin_lock_init(&domain->cache_lock); xa_init(&domain->iommu_array); spin_lock(&s2_domain->s1_lock); diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 11f0b856d74c..abce19e2ad6f 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -20,6 +20,7 @@ #include "iommu.h" #include "pasid.h" +#include "../iommu-pages.h" /* * Intel IOMMU system wide PASID name space: @@ -38,7 +39,7 @@ int intel_pasid_alloc_table(struct device *dev) { struct device_domain_info *info; struct pasid_table *pasid_table; - struct page *pages; + struct pasid_dir_entry *dir; u32 max_pasid = 0; int order, size; @@ -59,14 +60,13 @@ int intel_pasid_alloc_table(struct device *dev) size = max_pasid >> (PASID_PDE_SHIFT - 3); order = size ? get_order(size) : 0; - pages = alloc_pages_node(info->iommu->node, - GFP_KERNEL | __GFP_ZERO, order); - if (!pages) { + dir = iommu_alloc_pages_node(info->iommu->node, GFP_KERNEL, order); + if (!dir) { kfree(pasid_table); return -ENOMEM; } - pasid_table->table = page_address(pages); + pasid_table->table = dir; pasid_table->order = order; pasid_table->max_pasid = 1 << (order + PAGE_SHIFT + 3); info->pasid_table = pasid_table; @@ -97,10 +97,10 @@ void intel_pasid_free_table(struct device *dev) max_pde = pasid_table->max_pasid >> PASID_PDE_SHIFT; for (i = 0; i < max_pde; i++) { table = get_pasid_table_from_pde(&dir[i]); - free_pgtable_page(table); + iommu_free_page(table); } - free_pages((unsigned long)pasid_table->table, pasid_table->order); + iommu_free_pages(pasid_table->table, pasid_table->order); kfree(pasid_table); } @@ -146,7 +146,7 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) retry: entries = get_pasid_table_from_pde(&dir[dir_index]); if (!entries) { - entries = alloc_pgtable_page(info->iommu->node, GFP_ATOMIC); + entries = iommu_alloc_page_node(info->iommu->node, GFP_ATOMIC); if (!entries) return NULL; @@ -158,7 +158,7 @@ retry: */ if (cmpxchg64(&dir[dir_index].val, 0ULL, (u64)virt_to_phys(entries) | PASID_PTE_PRESENT)) { - free_pgtable_page(entries); + iommu_free_page(entries); goto retry; } if (!ecap_coherent(info->iommu->ecap)) { diff --git a/drivers/iommu/intel/perf.h b/drivers/iommu/intel/perf.h index fd6db8049d1a..df9a36942d64 100644 --- a/drivers/iommu/intel/perf.h +++ b/drivers/iommu/intel/perf.h @@ -11,7 +11,6 @@ enum latency_type { DMAR_LATENCY_INV_IOTLB = 0, DMAR_LATENCY_INV_DEVTLB, DMAR_LATENCY_INV_IEC, - DMAR_LATENCY_PRQ, DMAR_LATENCY_NUM }; diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index ee3b469e2da1..0e3a9b38bef2 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -22,57 +22,22 @@ #include "iommu.h" #include "pasid.h" #include "perf.h" +#include "../iommu-pages.h" #include "trace.h" static irqreturn_t prq_event_thread(int irq, void *d); -static DEFINE_XARRAY_ALLOC(pasid_private_array); -static int pasid_private_add(ioasid_t pasid, void *priv) -{ - return xa_alloc(&pasid_private_array, &pasid, priv, - XA_LIMIT(pasid, pasid), GFP_ATOMIC); -} - -static void pasid_private_remove(ioasid_t pasid) -{ - xa_erase(&pasid_private_array, pasid); -} - -static void *pasid_private_find(ioasid_t pasid) -{ - return xa_load(&pasid_private_array, pasid); -} - -static struct intel_svm_dev * -svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev) -{ - struct intel_svm_dev *sdev = NULL, *t; - - rcu_read_lock(); - list_for_each_entry_rcu(t, &svm->devs, list) { - if (t->dev == dev) { - sdev = t; - break; - } - } - rcu_read_unlock(); - - return sdev; -} - int intel_svm_enable_prq(struct intel_iommu *iommu) { struct iopf_queue *iopfq; - struct page *pages; int irq, ret; - pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); - if (!pages) { + iommu->prq = iommu_alloc_pages_node(iommu->node, GFP_KERNEL, PRQ_ORDER); + if (!iommu->prq) { pr_warn("IOMMU: %s: Failed to allocate page request queue\n", iommu->name); return -ENOMEM; } - iommu->prq = page_address(pages); irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PRQ + iommu->seq_id, iommu->node, iommu); if (irq <= 0) { @@ -117,7 +82,7 @@ free_hwirq: dmar_free_hwirq(irq); iommu->pr_irq = 0; free_prq: - free_pages((unsigned long)iommu->prq, PRQ_ORDER); + iommu_free_pages(iommu->prq, PRQ_ORDER); iommu->prq = NULL; return ret; @@ -140,7 +105,7 @@ int intel_svm_finish_prq(struct intel_iommu *iommu) iommu->iopf_queue = NULL; } - free_pages((unsigned long)iommu->prq, PRQ_ORDER); + iommu_free_pages(iommu->prq, PRQ_ORDER); iommu->prq = NULL; return 0; @@ -168,94 +133,32 @@ void intel_svm_check(struct intel_iommu *iommu) iommu->flags |= VTD_FLAG_SVM_CAPABLE; } -static void __flush_svm_range_dev(struct intel_svm *svm, - struct intel_svm_dev *sdev, - unsigned long address, - unsigned long pages, int ih) -{ - struct device_domain_info *info = dev_iommu_priv_get(sdev->dev); - - if (WARN_ON(!pages)) - return; - - qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih); - if (info->ats_enabled) { - qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, - svm->pasid, sdev->qdep, address, - order_base_2(pages)); - quirk_extra_dev_tlb_flush(info, address, order_base_2(pages), - svm->pasid, sdev->qdep); - } -} - -static void intel_flush_svm_range_dev(struct intel_svm *svm, - struct intel_svm_dev *sdev, - unsigned long address, - unsigned long pages, int ih) -{ - unsigned long shift = ilog2(__roundup_pow_of_two(pages)); - unsigned long align = (1ULL << (VTD_PAGE_SHIFT + shift)); - unsigned long start = ALIGN_DOWN(address, align); - unsigned long end = ALIGN(address + (pages << VTD_PAGE_SHIFT), align); - - while (start < end) { - __flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih); - start += align; - } -} - -static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, - unsigned long pages, int ih) -{ - struct intel_svm_dev *sdev; - - rcu_read_lock(); - list_for_each_entry_rcu(sdev, &svm->devs, list) - intel_flush_svm_range_dev(svm, sdev, address, pages, ih); - rcu_read_unlock(); -} - -static void intel_flush_svm_all(struct intel_svm *svm) -{ - struct device_domain_info *info; - struct intel_svm_dev *sdev; - - rcu_read_lock(); - list_for_each_entry_rcu(sdev, &svm->devs, list) { - info = dev_iommu_priv_get(sdev->dev); - - qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, 0, -1UL, 0); - if (info->ats_enabled) { - qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, - svm->pasid, sdev->qdep, - 0, 64 - VTD_PAGE_SHIFT); - quirk_extra_dev_tlb_flush(info, 0, 64 - VTD_PAGE_SHIFT, - svm->pasid, sdev->qdep); - } - } - rcu_read_unlock(); -} - /* Pages have been freed at this point */ static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, struct mm_struct *mm, unsigned long start, unsigned long end) { - struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); + struct dmar_domain *domain = container_of(mn, struct dmar_domain, notifier); - if (start == 0 && end == -1UL) { - intel_flush_svm_all(svm); + if (start == 0 && end == ULONG_MAX) { + cache_tag_flush_all(domain); return; } - intel_flush_svm_range(svm, start, - (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0); + /* + * The mm_types defines vm_end as the first byte after the end address, + * different from IOMMU subsystem using the last address of an address + * range. + */ + cache_tag_flush_range(domain, start, end - 1, 0); } static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) { - struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); - struct intel_svm_dev *sdev; + struct dmar_domain *domain = container_of(mn, struct dmar_domain, notifier); + struct dev_pasid_info *dev_pasid; + struct device_domain_info *info; + unsigned long flags; /* This might end up being called from exit_mmap(), *before* the page * tables are cleared. And __mmu_notifier_release() will delete us from @@ -269,157 +172,78 @@ static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) * page) so that we end up taking a fault that the hardware really * *has* to handle gracefully without affecting other processes. */ - rcu_read_lock(); - list_for_each_entry_rcu(sdev, &svm->devs, list) - intel_pasid_tear_down_entry(sdev->iommu, sdev->dev, - svm->pasid, true); - rcu_read_unlock(); + spin_lock_irqsave(&domain->lock, flags); + list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) { + info = dev_iommu_priv_get(dev_pasid->dev); + intel_pasid_tear_down_entry(info->iommu, dev_pasid->dev, + dev_pasid->pasid, true); + } + spin_unlock_irqrestore(&domain->lock, flags); + +} +static void intel_mm_free_notifier(struct mmu_notifier *mn) +{ + kfree(container_of(mn, struct dmar_domain, notifier)); } static const struct mmu_notifier_ops intel_mmuops = { .release = intel_mm_release, .arch_invalidate_secondary_tlbs = intel_arch_invalidate_secondary_tlbs, + .free_notifier = intel_mm_free_notifier, }; -static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, - struct intel_svm **rsvm, - struct intel_svm_dev **rsdev) -{ - struct intel_svm_dev *sdev = NULL; - struct intel_svm *svm; - - if (pasid == IOMMU_PASID_INVALID || pasid >= PASID_MAX) - return -EINVAL; - - svm = pasid_private_find(pasid); - if (IS_ERR(svm)) - return PTR_ERR(svm); - - if (!svm) - goto out; - - /* - * If we found svm for the PASID, there must be at least one device - * bond. - */ - if (WARN_ON(list_empty(&svm->devs))) - return -EINVAL; - sdev = svm_lookup_device_by_dev(svm, dev); - -out: - *rsvm = svm; - *rsdev = sdev; - - return 0; -} - static int intel_svm_set_dev_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid) { struct device_domain_info *info = dev_iommu_priv_get(dev); + struct dmar_domain *dmar_domain = to_dmar_domain(domain); struct intel_iommu *iommu = info->iommu; struct mm_struct *mm = domain->mm; - struct intel_svm_dev *sdev; - struct intel_svm *svm; + struct dev_pasid_info *dev_pasid; unsigned long sflags; + unsigned long flags; int ret = 0; - svm = pasid_private_find(pasid); - if (!svm) { - svm = kzalloc(sizeof(*svm), GFP_KERNEL); - if (!svm) - return -ENOMEM; - - svm->pasid = pasid; - svm->mm = mm; - INIT_LIST_HEAD_RCU(&svm->devs); - - svm->notifier.ops = &intel_mmuops; - ret = mmu_notifier_register(&svm->notifier, mm); - if (ret) { - kfree(svm); - return ret; - } - - ret = pasid_private_add(svm->pasid, svm); - if (ret) { - mmu_notifier_unregister(&svm->notifier, mm); - kfree(svm); - return ret; - } - } + dev_pasid = kzalloc(sizeof(*dev_pasid), GFP_KERNEL); + if (!dev_pasid) + return -ENOMEM; - sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); - if (!sdev) { - ret = -ENOMEM; - goto free_svm; - } + dev_pasid->dev = dev; + dev_pasid->pasid = pasid; - sdev->dev = dev; - sdev->iommu = iommu; - sdev->did = FLPT_DEFAULT_DID; - sdev->sid = PCI_DEVID(info->bus, info->devfn); - if (info->ats_enabled) { - sdev->qdep = info->ats_qdep; - if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) - sdev->qdep = 0; - } + ret = cache_tag_assign_domain(to_dmar_domain(domain), dev, pasid); + if (ret) + goto free_dev_pasid; /* Setup the pasid table: */ sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, pasid, FLPT_DEFAULT_DID, sflags); if (ret) - goto free_sdev; + goto unassign_tag; - list_add_rcu(&sdev->list, &svm->devs); + spin_lock_irqsave(&dmar_domain->lock, flags); + list_add(&dev_pasid->link_domain, &dmar_domain->dev_pasids); + spin_unlock_irqrestore(&dmar_domain->lock, flags); return 0; -free_sdev: - kfree(sdev); -free_svm: - if (list_empty(&svm->devs)) { - mmu_notifier_unregister(&svm->notifier, mm); - pasid_private_remove(pasid); - kfree(svm); - } +unassign_tag: + cache_tag_unassign_domain(to_dmar_domain(domain), dev, pasid); +free_dev_pasid: + kfree(dev_pasid); return ret; } -void intel_svm_remove_dev_pasid(struct device *dev, u32 pasid) -{ - struct intel_svm_dev *sdev; - struct intel_svm *svm; - struct mm_struct *mm; - - if (pasid_to_svm_sdev(dev, pasid, &svm, &sdev)) - return; - mm = svm->mm; - - if (sdev) { - list_del_rcu(&sdev->list); - kfree_rcu(sdev, rcu); - - if (list_empty(&svm->devs)) { - if (svm->notifier.ops) - mmu_notifier_unregister(&svm->notifier, mm); - pasid_private_remove(svm->pasid); - kfree(svm); - } - } -} - /* Page request queue descriptor */ struct page_req_dsc { union { struct { u64 type:8; u64 pasid_present:1; - u64 priv_data_present:1; - u64 rsvd:6; + u64 rsvd:7; u64 rid:16; u64 pasid:20; u64 exe_req:1; @@ -438,7 +262,8 @@ struct page_req_dsc { }; u64 qw_1; }; - u64 priv_data[2]; + u64 qw_2; + u64 qw_3; }; static bool is_canonical_address(u64 addr) @@ -572,24 +397,6 @@ static void intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; } - if (desc->priv_data_present) { - /* - * Set last page in group bit if private data is present, - * page response is required as it does for LPIG. - * iommu_report_device_fault() doesn't understand this vendor - * specific requirement thus we set last_page as a workaround. - */ - event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; - event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; - event.fault.prm.private_data[0] = desc->priv_data[0]; - event.fault.prm.private_data[1] = desc->priv_data[1]; - } else if (dmar_latency_enabled(iommu, DMAR_LATENCY_PRQ)) { - /* - * If the private data fields are not used by hardware, use it - * to monitor the prq handle latency. - */ - event.fault.prm.private_data[0] = ktime_to_ns(ktime_get()); - } iommu_report_device_fault(dev, &event); } @@ -597,39 +404,23 @@ static void intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, static void handle_bad_prq_event(struct intel_iommu *iommu, struct page_req_dsc *req, int result) { - struct qi_desc desc; + struct qi_desc desc = { }; pr_err("%s: Invalid page request: %08llx %08llx\n", iommu->name, ((unsigned long long *)req)[0], ((unsigned long long *)req)[1]); - /* - * Per VT-d spec. v3.0 ch7.7, system software must - * respond with page group response if private data - * is present (PDP) or last page in group (LPIG) bit - * is set. This is an additional VT-d feature beyond - * PCI ATS spec. - */ - if (!req->lpig && !req->priv_data_present) + if (!req->lpig) return; desc.qw0 = QI_PGRP_PASID(req->pasid) | QI_PGRP_DID(req->rid) | QI_PGRP_PASID_P(req->pasid_present) | - QI_PGRP_PDP(req->priv_data_present) | QI_PGRP_RESP_CODE(result) | QI_PGRP_RESP_TYPE; desc.qw1 = QI_PGRP_IDX(req->prg_index) | QI_PGRP_LPIG(req->lpig); - if (req->priv_data_present) { - desc.qw2 = req->priv_data[0]; - desc.qw3 = req->priv_data[1]; - } else { - desc.qw2 = 0; - desc.qw3 = 0; - } - qi_submit_sync(iommu, &desc, 1, 0); } @@ -697,7 +488,7 @@ bad_req: intel_svm_prq_report(iommu, dev, req); trace_prq_report(iommu, dev, req->qw_0, req->qw_1, - req->priv_data[0], req->priv_data[1], + req->qw_2, req->qw_3, iommu->prq_seq_number++); mutex_unlock(&iommu->iopf_lock); prq_advance: @@ -736,7 +527,7 @@ void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, struct intel_iommu *iommu = info->iommu; u8 bus = info->bus, devfn = info->devfn; struct iommu_fault_page_request *prm; - bool private_present; + struct qi_desc desc; bool pasid_present; bool last_page; u16 sid; @@ -744,42 +535,25 @@ void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, prm = &evt->fault.prm; sid = PCI_DEVID(bus, devfn); pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; - private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; - /* - * Per VT-d spec. v3.0 ch7.7, system software must respond - * with page group response if private data is present (PDP) - * or last page in group (LPIG) bit is set. This is an - * additional VT-d requirement beyond PCI ATS spec. - */ - if (last_page || private_present) { - struct qi_desc desc; - - desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | - QI_PGRP_PASID_P(pasid_present) | - QI_PGRP_PDP(private_present) | - QI_PGRP_RESP_CODE(msg->code) | - QI_PGRP_RESP_TYPE; - desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); - desc.qw2 = 0; - desc.qw3 = 0; - - if (private_present) { - desc.qw2 = prm->private_data[0]; - desc.qw3 = prm->private_data[1]; - } else if (prm->private_data[0]) { - dmar_latency_update(iommu, DMAR_LATENCY_PRQ, - ktime_to_ns(ktime_get()) - prm->private_data[0]); - } + desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | + QI_PGRP_PASID_P(pasid_present) | + QI_PGRP_RESP_CODE(msg->code) | + QI_PGRP_RESP_TYPE; + desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); + desc.qw2 = 0; + desc.qw3 = 0; - qi_submit_sync(iommu, &desc, 1, 0); - } + qi_submit_sync(iommu, &desc, 1, 0); } static void intel_svm_domain_free(struct iommu_domain *domain) { - kfree(to_dmar_domain(domain)); + struct dmar_domain *dmar_domain = to_dmar_domain(domain); + + /* dmar_domain free is deferred to the mmu free_notifier callback. */ + mmu_notifier_put(&dmar_domain->notifier); } static const struct iommu_domain_ops intel_svm_domain_ops = { @@ -787,14 +561,29 @@ static const struct iommu_domain_ops intel_svm_domain_ops = { .free = intel_svm_domain_free }; -struct iommu_domain *intel_svm_domain_alloc(void) +struct iommu_domain *intel_svm_domain_alloc(struct device *dev, + struct mm_struct *mm) { struct dmar_domain *domain; + int ret; domain = kzalloc(sizeof(*domain), GFP_KERNEL); if (!domain) - return NULL; + return ERR_PTR(-ENOMEM); + domain->domain.ops = &intel_svm_domain_ops; + domain->use_first_level = true; + INIT_LIST_HEAD(&domain->dev_pasids); + INIT_LIST_HEAD(&domain->cache_tags); + spin_lock_init(&domain->cache_lock); + spin_lock_init(&domain->lock); + + domain->notifier.ops = &intel_mmuops; + ret = mmu_notifier_register(&domain->notifier, mm); + if (ret) { + kfree(domain); + return ERR_PTR(ret); + } return &domain->domain; } diff --git a/drivers/iommu/intel/trace.h b/drivers/iommu/intel/trace.h index 93d96f93a89b..961ac1c1bc21 100644 --- a/drivers/iommu/intel/trace.h +++ b/drivers/iommu/intel/trace.h @@ -89,6 +89,103 @@ TRACE_EVENT(prq_report, __entry->dw1, __entry->dw2, __entry->dw3) ) ); + +DECLARE_EVENT_CLASS(cache_tag_log, + TP_PROTO(struct cache_tag *tag), + TP_ARGS(tag), + TP_STRUCT__entry( + __string(iommu, tag->iommu->name) + __string(dev, dev_name(tag->dev)) + __field(u16, type) + __field(u16, domain_id) + __field(u32, pasid) + __field(u32, users) + ), + TP_fast_assign( + __assign_str(iommu, tag->iommu->name); + __assign_str(dev, dev_name(tag->dev)); + __entry->type = tag->type; + __entry->domain_id = tag->domain_id; + __entry->pasid = tag->pasid; + __entry->users = tag->users; + ), + TP_printk("%s/%s type %s did %d pasid %d ref %d", + __get_str(iommu), __get_str(dev), + __print_symbolic(__entry->type, + { CACHE_TAG_IOTLB, "iotlb" }, + { CACHE_TAG_DEVTLB, "devtlb" }, + { CACHE_TAG_NESTING_IOTLB, "nesting_iotlb" }, + { CACHE_TAG_NESTING_DEVTLB, "nesting_devtlb" }), + __entry->domain_id, __entry->pasid, __entry->users + ) +); + +DEFINE_EVENT(cache_tag_log, cache_tag_assign, + TP_PROTO(struct cache_tag *tag), + TP_ARGS(tag) +); + +DEFINE_EVENT(cache_tag_log, cache_tag_unassign, + TP_PROTO(struct cache_tag *tag), + TP_ARGS(tag) +); + +DEFINE_EVENT(cache_tag_log, cache_tag_flush_all, + TP_PROTO(struct cache_tag *tag), + TP_ARGS(tag) +); + +DECLARE_EVENT_CLASS(cache_tag_flush, + TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end, + unsigned long addr, unsigned long pages, unsigned long mask), + TP_ARGS(tag, start, end, addr, pages, mask), + TP_STRUCT__entry( + __string(iommu, tag->iommu->name) + __string(dev, dev_name(tag->dev)) + __field(u16, type) + __field(u16, domain_id) + __field(u32, pasid) + __field(unsigned long, start) + __field(unsigned long, end) + __field(unsigned long, addr) + __field(unsigned long, pages) + __field(unsigned long, mask) + ), + TP_fast_assign( + __assign_str(iommu, tag->iommu->name); + __assign_str(dev, dev_name(tag->dev)); + __entry->type = tag->type; + __entry->domain_id = tag->domain_id; + __entry->pasid = tag->pasid; + __entry->start = start; + __entry->end = end; + __entry->addr = addr; + __entry->pages = pages; + __entry->mask = mask; + ), + TP_printk("%s %s[%d] type %s did %d [0x%lx-0x%lx] addr 0x%lx pages 0x%lx mask 0x%lx", + __get_str(iommu), __get_str(dev), __entry->pasid, + __print_symbolic(__entry->type, + { CACHE_TAG_IOTLB, "iotlb" }, + { CACHE_TAG_DEVTLB, "devtlb" }, + { CACHE_TAG_NESTING_IOTLB, "nesting_iotlb" }, + { CACHE_TAG_NESTING_DEVTLB, "nesting_devtlb" }), + __entry->domain_id, __entry->start, __entry->end, + __entry->addr, __entry->pages, __entry->mask + ) +); + +DEFINE_EVENT(cache_tag_flush, cache_tag_flush_range, + TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end, + unsigned long addr, unsigned long pages, unsigned long mask), + TP_ARGS(tag, start, end, addr, pages, mask) +); + +DEFINE_EVENT(cache_tag_flush, cache_tag_flush_range_np, + TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end, + unsigned long addr, unsigned long pages, unsigned long mask), + TP_ARGS(tag, start, end, addr, pages, mask) +); #endif /* _TRACE_INTEL_IOMMU_H */ /* This part must be outside protection */ diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index f7828a7aad41..3d23b924cec1 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -21,6 +21,7 @@ #include <asm/barrier.h> #include "io-pgtable-arm.h" +#include "iommu-pages.h" #define ARM_LPAE_MAX_ADDR_BITS 52 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 @@ -198,14 +199,10 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, VM_BUG_ON((gfp & __GFP_HIGHMEM)); - if (cfg->alloc) { + if (cfg->alloc) pages = cfg->alloc(cookie, size, gfp); - } else { - struct page *p; - - p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); - pages = p ? page_address(p) : NULL; - } + else + pages = iommu_alloc_pages_node(dev_to_node(dev), gfp, order); if (!pages) return NULL; @@ -233,7 +230,7 @@ out_free: if (cfg->free) cfg->free(cookie, pages, size); else - free_pages((unsigned long)pages, order); + iommu_free_pages(pages, order); return NULL; } @@ -249,7 +246,7 @@ static void __arm_lpae_free_pages(void *pages, size_t size, if (cfg->free) cfg->free(cookie, pages, size); else - free_pages((unsigned long)pages, get_order(size)); + iommu_free_pages(pages, get_order(size)); } static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c index 74b1ef2b96be..ad28031e1e93 100644 --- a/drivers/iommu/io-pgtable-dart.c +++ b/drivers/iommu/io-pgtable-dart.c @@ -23,6 +23,7 @@ #include <linux/types.h> #include <asm/barrier.h> +#include "iommu-pages.h" #define DART1_MAX_ADDR_BITS 36 @@ -106,18 +107,12 @@ static phys_addr_t iopte_to_paddr(dart_iopte pte, return paddr; } -static void *__dart_alloc_pages(size_t size, gfp_t gfp, - struct io_pgtable_cfg *cfg) +static void *__dart_alloc_pages(size_t size, gfp_t gfp) { int order = get_order(size); - struct page *p; VM_BUG_ON((gfp & __GFP_HIGHMEM)); - p = alloc_pages(gfp | __GFP_ZERO, order); - if (!p) - return NULL; - - return page_address(p); + return iommu_alloc_pages(gfp, order); } static int dart_init_pte(struct dart_io_pgtable *data, @@ -262,13 +257,13 @@ static int dart_map_pages(struct io_pgtable_ops *ops, unsigned long iova, /* no L2 table present */ if (!pte) { - cptep = __dart_alloc_pages(tblsz, gfp, cfg); + cptep = __dart_alloc_pages(tblsz, gfp); if (!cptep) return -ENOMEM; pte = dart_install_table(cptep, ptep, 0, data); if (pte) - free_pages((unsigned long)cptep, get_order(tblsz)); + iommu_free_pages(cptep, get_order(tblsz)); /* L2 table is present (now) */ pte = READ_ONCE(*ptep); @@ -419,8 +414,7 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) cfg->apple_dart_cfg.n_ttbrs = 1 << data->tbl_bits; for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i) { - data->pgd[i] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL, - cfg); + data->pgd[i] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL); if (!data->pgd[i]) goto out_free_data; cfg->apple_dart_cfg.ttbr[i] = virt_to_phys(data->pgd[i]); @@ -429,9 +423,10 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) return &data->iop; out_free_data: - while (--i >= 0) - free_pages((unsigned long)data->pgd[i], - get_order(DART_GRANULE(data))); + while (--i >= 0) { + iommu_free_pages(data->pgd[i], + get_order(DART_GRANULE(data))); + } kfree(data); return NULL; } @@ -439,6 +434,7 @@ out_free_data: static void apple_dart_free_pgtable(struct io_pgtable *iop) { struct dart_io_pgtable *data = io_pgtable_to_data(iop); + int order = get_order(DART_GRANULE(data)); dart_iopte *ptep, *end; int i; @@ -449,15 +445,10 @@ static void apple_dart_free_pgtable(struct io_pgtable *iop) while (ptep != end) { dart_iopte pte = *ptep++; - if (pte) { - unsigned long page = - (unsigned long)iopte_deref(pte, data); - - free_pages(page, get_order(DART_GRANULE(data))); - } + if (pte) + iommu_free_pages(iopte_deref(pte, data), order); } - free_pages((unsigned long)data->pgd[i], - get_order(DART_GRANULE(data))); + iommu_free_pages(data->pgd[i], order); } kfree(data); diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h new file mode 100644 index 000000000000..82ebf0033081 --- /dev/null +++ b/drivers/iommu/iommu-pages.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2024, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +#ifndef __IOMMU_PAGES_H +#define __IOMMU_PAGES_H + +#include <linux/vmstat.h> +#include <linux/gfp.h> +#include <linux/mm.h> + +/* + * All page allocations that should be reported to as "iommu-pagetables" to + * userspace must use one of the functions below. This includes allocations of + * page-tables and other per-iommu_domain configuration structures. + * + * This is necessary for the proper accounting as IOMMU state can be rather + * large, i.e. multiple gigabytes in size. + */ + +/** + * __iommu_alloc_account - account for newly allocated page. + * @page: head struct page of the page. + * @order: order of the page + */ +static inline void __iommu_alloc_account(struct page *page, int order) +{ + const long pgcnt = 1l << order; + + mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt); + mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt); +} + +/** + * __iommu_free_account - account a page that is about to be freed. + * @page: head struct page of the page. + * @order: order of the page + */ +static inline void __iommu_free_account(struct page *page, int order) +{ + const long pgcnt = 1l << order; + + mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt); + mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt); +} + +/** + * __iommu_alloc_pages - allocate a zeroed page of a given order. + * @gfp: buddy allocator flags + * @order: page order + * + * returns the head struct page of the allocated page. + */ +static inline struct page *__iommu_alloc_pages(gfp_t gfp, int order) +{ + struct page *page; + + page = alloc_pages(gfp | __GFP_ZERO, order); + if (unlikely(!page)) + return NULL; + + __iommu_alloc_account(page, order); + + return page; +} + +/** + * __iommu_free_pages - free page of a given order + * @page: head struct page of the page + * @order: page order + */ +static inline void __iommu_free_pages(struct page *page, int order) +{ + if (!page) + return; + + __iommu_free_account(page, order); + __free_pages(page, order); +} + +/** + * iommu_alloc_pages_node - allocate a zeroed page of a given order from + * specific NUMA node. + * @nid: memory NUMA node id + * @gfp: buddy allocator flags + * @order: page order + * + * returns the virtual address of the allocated page + */ +static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order) +{ + struct page *page = alloc_pages_node(nid, gfp | __GFP_ZERO, order); + + if (unlikely(!page)) + return NULL; + + __iommu_alloc_account(page, order); + + return page_address(page); +} + +/** + * iommu_alloc_pages - allocate a zeroed page of a given order + * @gfp: buddy allocator flags + * @order: page order + * + * returns the virtual address of the allocated page + */ +static inline void *iommu_alloc_pages(gfp_t gfp, int order) +{ + struct page *page = __iommu_alloc_pages(gfp, order); + + if (unlikely(!page)) + return NULL; + + return page_address(page); +} + +/** + * iommu_alloc_page_node - allocate a zeroed page at specific NUMA node. + * @nid: memory NUMA node id + * @gfp: buddy allocator flags + * + * returns the virtual address of the allocated page + */ +static inline void *iommu_alloc_page_node(int nid, gfp_t gfp) +{ + return iommu_alloc_pages_node(nid, gfp, 0); +} + +/** + * iommu_alloc_page - allocate a zeroed page + * @gfp: buddy allocator flags + * + * returns the virtual address of the allocated page + */ +static inline void *iommu_alloc_page(gfp_t gfp) +{ + return iommu_alloc_pages(gfp, 0); +} + +/** + * iommu_free_pages - free page of a given order + * @virt: virtual address of the page to be freed. + * @order: page order + */ +static inline void iommu_free_pages(void *virt, int order) +{ + if (!virt) + return; + + __iommu_free_pages(virt_to_page(virt), order); +} + +/** + * iommu_free_page - free page + * @virt: virtual address of the page to be freed. + */ +static inline void iommu_free_page(void *virt) +{ + iommu_free_pages(virt, 0); +} + +/** + * iommu_put_pages_list - free a list of pages. + * @page: the head of the lru list to be freed. + * + * There are no locking requirement for these pages, as they are going to be + * put on a free list as soon as refcount reaches 0. Pages are put on this LRU + * list once they are removed from the IOMMU page tables. However, they can + * still be access through debugfs. + */ +static inline void iommu_put_pages_list(struct list_head *page) +{ + while (!list_empty(page)) { + struct page *p = list_entry(page->prev, struct page, lru); + + list_del(&p->lru); + __iommu_free_account(p, 0); + put_page(p); + } +} + +#endif /* __IOMMU_PAGES_H */ diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 640acc804e8c..18a35e798b72 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -108,8 +108,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm /* Allocate a new domain and set it on device pasid. */ domain = iommu_sva_domain_alloc(dev, mm); - if (!domain) { - ret = -ENOMEM; + if (IS_ERR(domain)) { + ret = PTR_ERR(domain); goto out_free_handle; } @@ -283,9 +283,15 @@ struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, const struct iommu_ops *ops = dev_iommu_ops(dev); struct iommu_domain *domain; - domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); - if (!domain) - return NULL; + if (ops->domain_alloc_sva) { + domain = ops->domain_alloc_sva(dev, mm); + if (IS_ERR(domain)) + return domain; + } else { + domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); + if (!domain) + return ERR_PTR(-ENOMEM); + } domain->type = IOMMU_DOMAIN_SVA; mmgrab(mm); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index a95a483def2d..9df7cc75c1bc 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -581,10 +581,11 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list if (list_empty(&group->entry)) list_add_tail(&group->entry, group_list); } - mutex_unlock(&group->mutex); - if (dev_is_pci(dev)) - iommu_dma_set_pci_32bit_workaround(dev); + if (group->default_domain) + iommu_setup_dma_ops(dev); + + mutex_unlock(&group->mutex); return 0; @@ -1828,6 +1829,8 @@ int bus_iommu_probe(const struct bus_type *bus) mutex_unlock(&group->mutex); return ret; } + for_each_group_device(group, gdev) + iommu_setup_dma_ops(gdev->dev); mutex_unlock(&group->mutex); /* @@ -3066,18 +3069,9 @@ static ssize_t iommu_group_store_type(struct iommu_group *group, if (ret) goto out_unlock; - /* - * Release the mutex here because ops->probe_finalize() call-back of - * some vendor IOMMU drivers calls arm_iommu_attach_device() which - * in-turn might call back into IOMMU core code, where it tries to take - * group->mutex, resulting in a deadlock. - */ - mutex_unlock(&group->mutex); - /* Make sure dma_ops is appropriatley set */ for_each_group_device(group, gdev) - iommu_group_do_probe_finalize(gdev->dev); - return count; + iommu_setup_dma_ops(gdev->dev); out_unlock: mutex_unlock(&group->mutex); @@ -3317,27 +3311,39 @@ EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed); static int __iommu_set_group_pasid(struct iommu_domain *domain, struct iommu_group *group, ioasid_t pasid) { - struct group_device *device; - int ret = 0; + struct group_device *device, *last_gdev; + int ret; for_each_group_device(group, device) { ret = domain->ops->set_dev_pasid(domain, device->dev, pasid); if (ret) - break; + goto err_revert; } + return 0; + +err_revert: + last_gdev = device; + for_each_group_device(group, device) { + const struct iommu_ops *ops = dev_iommu_ops(device->dev); + + if (device == last_gdev) + break; + ops->remove_dev_pasid(device->dev, pasid, domain); + } return ret; } static void __iommu_remove_group_pasid(struct iommu_group *group, - ioasid_t pasid) + ioasid_t pasid, + struct iommu_domain *domain) { struct group_device *device; const struct iommu_ops *ops; for_each_group_device(group, device) { ops = dev_iommu_ops(device->dev); - ops->remove_dev_pasid(device->dev, pasid); + ops->remove_dev_pasid(device->dev, pasid, domain); } } @@ -3383,10 +3389,8 @@ int iommu_attach_device_pasid(struct iommu_domain *domain, } ret = __iommu_set_group_pasid(domain, group, pasid); - if (ret) { - __iommu_remove_group_pasid(group, pasid); + if (ret) xa_erase(&group->pasid_array, pasid); - } out_unlock: mutex_unlock(&group->mutex); return ret; @@ -3409,7 +3413,7 @@ void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev, struct iommu_group *group = dev->iommu_group; mutex_lock(&group->mutex); - __iommu_remove_group_pasid(group, pasid); + __iommu_remove_group_pasid(group, pasid, domain); WARN_ON(xa_erase(&group->pasid_array, pasid) != domain); mutex_unlock(&group->mutex); } diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index 056fec6991bc..c2443659812a 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -154,7 +154,10 @@ int __init irq_remap_enable_fault_handling(void) if (!remap_ops->enable_faulting) return -ENODEV; - return remap_ops->enable_faulting(); + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "dmar:enable_fault_handling", + remap_ops->enable_faulting, NULL); + + return remap_ops->enable_faulting(smp_processor_id()); } void panic_if_irq_remap(const char *msg) diff --git a/drivers/iommu/irq_remapping.h b/drivers/iommu/irq_remapping.h index 8c89cb947cdb..0d6f140b5e01 100644 --- a/drivers/iommu/irq_remapping.h +++ b/drivers/iommu/irq_remapping.h @@ -41,7 +41,7 @@ struct irq_remap_ops { int (*reenable)(int); /* Enable fault handling */ - int (*enable_faulting)(void); + int (*enable_faulting)(unsigned int); }; extern struct irq_remap_ops intel_irq_remap_ops; diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index da79d9f4cf63..4b369419b32c 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -26,6 +26,8 @@ #include <linux/slab.h> #include <linux/spinlock.h> +#include "iommu-pages.h" + /** MMU register offsets */ #define RK_MMU_DTE_ADDR 0x00 /* Directory table address */ #define RK_MMU_STATUS 0x04 @@ -727,14 +729,14 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, if (rk_dte_is_pt_valid(dte)) goto done; - page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | rk_ops->gfp_flags); + page_table = iommu_alloc_page(GFP_ATOMIC | rk_ops->gfp_flags); if (!page_table) return ERR_PTR(-ENOMEM); pt_dma = dma_map_single(dma_dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(dma_dev, pt_dma)) { dev_err(dma_dev, "DMA mapping error while allocating page table\n"); - free_page((unsigned long)page_table); + iommu_free_page(page_table); return ERR_PTR(-ENOMEM); } @@ -1061,7 +1063,7 @@ static struct iommu_domain *rk_iommu_domain_alloc_paging(struct device *dev) * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. * Allocate one 4 KiB page for each table. */ - rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | rk_ops->gfp_flags); + rk_domain->dt = iommu_alloc_page(GFP_KERNEL | rk_ops->gfp_flags); if (!rk_domain->dt) goto err_free_domain; @@ -1083,7 +1085,7 @@ static struct iommu_domain *rk_iommu_domain_alloc_paging(struct device *dev) return &rk_domain->domain; err_free_dt: - free_page((unsigned long)rk_domain->dt); + iommu_free_page(rk_domain->dt); err_free_domain: kfree(rk_domain); @@ -1104,13 +1106,13 @@ static void rk_iommu_domain_free(struct iommu_domain *domain) u32 *page_table = phys_to_virt(pt_phys); dma_unmap_single(dma_dev, pt_phys, SPAGE_SIZE, DMA_TO_DEVICE); - free_page((unsigned long)page_table); + iommu_free_page(page_table); } } dma_unmap_single(dma_dev, rk_domain->dt_dma, SPAGE_SIZE, DMA_TO_DEVICE); - free_page((unsigned long)rk_domain->dt); + iommu_free_page(rk_domain->dt); kfree(rk_domain); } diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index 9a5196f523de..d8eaa7ea380b 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -695,11 +695,6 @@ static size_t s390_iommu_unmap_pages(struct iommu_domain *domain, return size; } -static void s390_iommu_probe_finalize(struct device *dev) -{ - iommu_setup_dma_ops(dev, 0, U64_MAX); -} - struct zpci_iommu_ctrs *zpci_get_iommu_ctrs(struct zpci_dev *zdev) { if (!zdev || !zdev->s390_domain) @@ -785,7 +780,6 @@ static const struct iommu_ops s390_iommu_ops = { .capable = s390_iommu_capable, .domain_alloc_paging = s390_domain_alloc_paging, .probe_device = s390_iommu_probe_device, - .probe_finalize = s390_iommu_probe_finalize, .release_device = s390_iommu_release_device, .device_group = generic_device_group, .pgsize_bitmap = SZ_4K, diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index decd52cba998..c519b991749d 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -26,6 +26,8 @@ #include <linux/spinlock.h> #include <linux/types.h> +#include "iommu-pages.h" + #define IOMMU_RESET_REG 0x010 #define IOMMU_RESET_RELEASE_ALL 0xffffffff #define IOMMU_ENABLE_REG 0x020 @@ -679,8 +681,7 @@ sun50i_iommu_domain_alloc_paging(struct device *dev) if (!sun50i_domain) return NULL; - sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(DT_SIZE)); + sun50i_domain->dt = iommu_alloc_pages(GFP_KERNEL, get_order(DT_SIZE)); if (!sun50i_domain->dt) goto err_free_domain; @@ -702,7 +703,7 @@ static void sun50i_iommu_domain_free(struct iommu_domain *domain) { struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); - free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE)); + iommu_free_pages(sun50i_domain->dt, get_order(DT_SIZE)); sun50i_domain->dt = NULL; kfree(sun50i_domain); diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 14e525bd0d9b..f86c7ae91814 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -19,6 +19,8 @@ #include <soc/tegra/ahb.h> #include <soc/tegra/mc.h> +#include "iommu-pages.h" + struct tegra_smmu_group { struct list_head list; struct tegra_smmu *smmu; @@ -282,7 +284,7 @@ static struct iommu_domain *tegra_smmu_domain_alloc_paging(struct device *dev) as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; - as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); + as->pd = __iommu_alloc_pages(GFP_KERNEL | __GFP_DMA, 0); if (!as->pd) { kfree(as); return NULL; @@ -290,7 +292,7 @@ static struct iommu_domain *tegra_smmu_domain_alloc_paging(struct device *dev) as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL); if (!as->count) { - __free_page(as->pd); + __iommu_free_pages(as->pd, 0); kfree(as); return NULL; } @@ -298,7 +300,7 @@ static struct iommu_domain *tegra_smmu_domain_alloc_paging(struct device *dev) as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); if (!as->pts) { kfree(as->count); - __free_page(as->pd); + __iommu_free_pages(as->pd, 0); kfree(as); return NULL; } @@ -599,14 +601,14 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT, DMA_TO_DEVICE); if (dma_mapping_error(smmu->dev, dma)) { - __free_page(page); + __iommu_free_pages(page, 0); return NULL; } if (!smmu_dma_addr_valid(smmu, dma)) { dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT, DMA_TO_DEVICE); - __free_page(page); + __iommu_free_pages(page, 0); return NULL; } @@ -649,7 +651,7 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) tegra_smmu_set_pde(as, iova, 0); dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); - __free_page(page); + __iommu_free_pages(page, 0); as->pts[pde] = NULL; } } @@ -688,7 +690,7 @@ static struct page *as_get_pde_page(struct tegra_smmu_as *as, if (gfpflags_allow_blocking(gfp)) spin_unlock_irqrestore(&as->lock, *flags); - page = alloc_page(gfp | __GFP_DMA | __GFP_ZERO); + page = __iommu_alloc_pages(gfp | __GFP_DMA, 0); if (gfpflags_allow_blocking(gfp)) spin_lock_irqsave(&as->lock, *flags); @@ -700,7 +702,7 @@ static struct page *as_get_pde_page(struct tegra_smmu_as *as, */ if (as->pts[pde]) { if (page) - __free_page(page); + __iommu_free_pages(page, 0); page = as->pts[pde]; } diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 04048f64a2c0..8e776f6c6e35 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -1025,15 +1025,6 @@ err_free_dev: return ERR_PTR(ret); } -static void viommu_probe_finalize(struct device *dev) -{ -#ifndef CONFIG_ARCH_HAS_SETUP_DMA_OPS - /* First clear the DMA ops in case we're switching from a DMA domain */ - set_dma_ops(dev, NULL); - iommu_setup_dma_ops(dev, 0, U64_MAX); -#endif -} - static void viommu_release_device(struct device *dev) { struct viommu_endpoint *vdev = dev_iommu_priv_get(dev); @@ -1073,7 +1064,6 @@ static struct iommu_ops viommu_ops = { .capable = viommu_capable, .domain_alloc = viommu_domain_alloc, .probe_device = viommu_probe_device, - .probe_finalize = viommu_probe_finalize, .release_device = viommu_release_device, .device_group = viommu_device_group, .get_resv_regions = viommu_get_resv_regions, diff --git a/drivers/macintosh/Kconfig b/drivers/macintosh/Kconfig index a0e717a986dc..fb38f684444f 100644 --- a/drivers/macintosh/Kconfig +++ b/drivers/macintosh/Kconfig @@ -262,7 +262,7 @@ config SENSORS_AMS will be called ams. config SENSORS_AMS_PMU - bool "PMU variant" + bool "PMU variant" if SENSORS_AMS_I2C depends on SENSORS_AMS && ADB_PMU default y help diff --git a/drivers/macintosh/macio-adb.c b/drivers/macintosh/macio-adb.c index 779f1268286e..19c63959ebed 100644 --- a/drivers/macintosh/macio-adb.c +++ b/drivers/macintosh/macio-adb.c @@ -83,35 +83,32 @@ struct adb_driver macio_adb_driver = { int macio_probe(void) { - struct device_node *np; + struct device_node *np __free(device_node) = + of_find_compatible_node(NULL, "adb", "chrp,adb0"); - np = of_find_compatible_node(NULL, "adb", "chrp,adb0"); - if (np) { - of_node_put(np); + if (np) return 0; - } + return -ENODEV; } int macio_init(void) { - struct device_node *adbs; + struct device_node *adbs __free(device_node) = + of_find_compatible_node(NULL, "adb", "chrp,adb0"); struct resource r; unsigned int irq; - adbs = of_find_compatible_node(NULL, "adb", "chrp,adb0"); if (!adbs) return -ENXIO; - if (of_address_to_resource(adbs, 0, &r)) { - of_node_put(adbs); + if (of_address_to_resource(adbs, 0, &r)) return -ENXIO; - } + adb = ioremap(r.start, sizeof(struct adb_regs)); - if (!adb) { - of_node_put(adbs); + if (!adb) return -ENOMEM; - } + out_8(&adb->ctrl.r, 0); out_8(&adb->intr.r, 0); @@ -121,7 +118,6 @@ int macio_init(void) out_8(&adb->autopoll.r, APE); irq = irq_of_parse_and_map(adbs, 0); - of_node_put(adbs); if (request_irq(irq, macio_adb_interrupt, 0, "ADB", (void *)0)) { iounmap(adb); printk(KERN_ERR "ADB: can't get irq %d\n", irq); diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig index 42940108a187..3b8842c4a340 100644 --- a/drivers/mailbox/Kconfig +++ b/drivers/mailbox/Kconfig @@ -23,6 +23,18 @@ config ARM_MHU_V2 Say Y here if you want to build the ARM MHUv2 controller driver, which provides unidirectional mailboxes between processing elements. +config ARM_MHU_V3 + tristate "ARM MHUv3 Mailbox" + depends on HAS_IOMEM || COMPILE_TEST + depends on OF + help + Say Y here if you want to build the ARM MHUv3 controller driver, + which provides unidirectional mailboxes between processing elements. + + ARM MHUv3 controllers can implement a varying number of extensions + that provides different means of transports: supported extensions + will be discovered and possibly managed at probe-time. + config IMX_MBOX tristate "i.MX Mailbox" depends on ARCH_MXC || COMPILE_TEST @@ -68,15 +80,6 @@ config OMAP2PLUS_MBOX OMAP2/3; or IPU, IVA HD and DSP in OMAP4/5. Say Y here if you want to use OMAP2+ Mailbox framework support. -config OMAP_MBOX_KFIFO_SIZE - int "Mailbox kfifo default buffer size (bytes)" - depends on OMAP2PLUS_MBOX - default 256 - help - Specify the default size of mailbox's kfifo buffers (bytes). - This can also be changed at runtime (via the mbox_kfifo_size - module parameter). - config ROCKCHIP_MBOX bool "Rockchip Soc Integrated Mailbox Support" depends on ARCH_ROCKCHIP || COMPILE_TEST diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile index 18793e6caa2f..5cf2f54debaf 100644 --- a/drivers/mailbox/Makefile +++ b/drivers/mailbox/Makefile @@ -9,6 +9,8 @@ obj-$(CONFIG_ARM_MHU) += arm_mhu.o arm_mhu_db.o obj-$(CONFIG_ARM_MHU_V2) += arm_mhuv2.o +obj-$(CONFIG_ARM_MHU_V3) += arm_mhuv3.o + obj-$(CONFIG_IMX_MBOX) += imx-mailbox.o obj-$(CONFIG_ARMADA_37XX_RWTM_MBOX) += armada-37xx-rwtm-mailbox.o diff --git a/drivers/mailbox/arm_mhuv3.c b/drivers/mailbox/arm_mhuv3.c new file mode 100644 index 000000000000..b97e79a5870f --- /dev/null +++ b/drivers/mailbox/arm_mhuv3.c @@ -0,0 +1,1103 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ARM Message Handling Unit Version 3 (MHUv3) driver. + * + * Copyright (C) 2024 ARM Ltd. + * + * Based on ARM MHUv2 driver. + */ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/bits.h> +#include <linux/cleanup.h> +#include <linux/device.h> +#include <linux/interrupt.h> +#include <linux/mailbox_controller.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/platform_device.h> +#include <linux/spinlock.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/types.h> + +/* ====== MHUv3 Registers ====== */ + +/* Maximum number of Doorbell channel windows */ +#define MHUV3_DBCW_MAX 128 +/* Number of DBCH combined interrupt status registers */ +#define MHUV3_DBCH_CMB_INT_ST_REG_CNT 4 + +/* Number of FFCH combined interrupt status registers */ +#define MHUV3_FFCH_CMB_INT_ST_REG_CNT 2 + +#define MHUV3_FLAG_BITS 32 + +/* Not a typo ... */ +#define MHUV3_MAJOR_VERSION 2 + +enum { + MHUV3_MBOX_CELL_TYPE, + MHUV3_MBOX_CELL_CHWN, + MHUV3_MBOX_CELL_PARAM, + MHUV3_MBOX_CELLS +}; + +/* Padding bitfields/fields represents hole in the regs MMIO */ + +/* CTRL_Page */ +struct blk_id { +#define id GENMASK(3, 0) + u32 val; +} __packed; + +struct feat_spt0 { +#define dbe_spt GENMASK(3, 0) +#define fe_spt GENMASK(7, 4) +#define fce_spt GENMASK(11, 8) + u32 val; +} __packed; + +struct feat_spt1 { +#define auto_op_spt GENMASK(3, 0) + u32 val; +} __packed; + +struct dbch_cfg0 { +#define num_dbch GENMASK(7, 0) + u32 val; +} __packed; + +struct ffch_cfg0 { +#define num_ffch GENMASK(7, 0) +#define x8ba_spt BIT(8) +#define x16ba_spt BIT(9) +#define x32ba_spt BIT(10) +#define x64ba_spt BIT(11) +#define ffch_depth GENMASK(25, 16) + u32 val; +} __packed; + +struct fch_cfg0 { +#define num_fch GENMASK(9, 0) +#define fcgi_spt BIT(10) // MBX-only +#define num_fcg GENMASK(15, 11) +#define num_fch_per_grp GENMASK(20, 16) +#define fch_ws GENMASK(28, 21) + u32 val; +} __packed; + +struct ctrl { +#define op_req BIT(0) +#define ch_op_mask BIT(1) + u32 val; +} __packed; + +struct fch_ctrl { +#define _int_en BIT(2) + u32 val; +} __packed; + +struct iidr { +#define implementer GENMASK(11, 0) +#define revision GENMASK(15, 12) +#define variant GENMASK(19, 16) +#define product_id GENMASK(31, 20) + u32 val; +} __packed; + +struct aidr { +#define arch_minor_rev GENMASK(3, 0) +#define arch_major_rev GENMASK(7, 4) + u32 val; +} __packed; + +struct ctrl_page { + struct blk_id blk_id; + u8 pad[12]; + struct feat_spt0 feat_spt0; + struct feat_spt1 feat_spt1; + u8 pad1[8]; + struct dbch_cfg0 dbch_cfg0; + u8 pad2[12]; + struct ffch_cfg0 ffch_cfg0; + u8 pad3[12]; + struct fch_cfg0 fch_cfg0; + u8 pad4[188]; + struct ctrl x_ctrl; + /*-- MBX-only registers --*/ + u8 pad5[60]; + struct fch_ctrl fch_ctrl; + u32 fcg_int_en; + u8 pad6[696]; + /*-- End of MBX-only ---- */ + u32 dbch_int_st[MHUV3_DBCH_CMB_INT_ST_REG_CNT]; + u32 ffch_int_st[MHUV3_FFCH_CMB_INT_ST_REG_CNT]; + /*-- MBX-only registers --*/ + u8 pad7[88]; + u32 fcg_int_st; + u8 pad8[12]; + u32 fcg_grp_int_st[32]; + u8 pad9[2760]; + /*-- End of MBX-only ---- */ + struct iidr iidr; + struct aidr aidr; + u32 imp_def_id[12]; +} __packed; + +/* DBCW_Page */ + +struct xbcw_ctrl { +#define comb_en BIT(0) + u32 val; +} __packed; + +struct pdbcw_int { +#define tfr_ack BIT(0) + u32 val; +} __packed; + +struct pdbcw_page { + u32 st; + u8 pad[8]; + u32 set; + struct pdbcw_int int_st; + struct pdbcw_int int_clr; + struct pdbcw_int int_en; + struct xbcw_ctrl ctrl; +} __packed; + +struct mdbcw_page { + u32 st; + u32 st_msk; + u32 clr; + u8 pad[4]; + u32 msk_st; + u32 msk_set; + u32 msk_clr; + struct xbcw_ctrl ctrl; +} __packed; + +struct dummy_page { + u8 pad[SZ_4K]; +} __packed; + +struct mhu3_pbx_frame_reg { + struct ctrl_page ctrl; + struct pdbcw_page dbcw[MHUV3_DBCW_MAX]; + struct dummy_page ffcw; + struct dummy_page fcw; + u8 pad[SZ_4K * 11]; + struct dummy_page impdef; +} __packed; + +struct mhu3_mbx_frame_reg { + struct ctrl_page ctrl; + struct mdbcw_page dbcw[MHUV3_DBCW_MAX]; + struct dummy_page ffcw; + struct dummy_page fcw; + u8 pad[SZ_4K * 11]; + struct dummy_page impdef; +} __packed; + +/* Macro for reading a bitmask within a physically mapped packed struct */ +#define readl_relaxed_bitmask(_regptr, _bitmask) \ + ({ \ + unsigned long _rval; \ + _rval = readl_relaxed(_regptr); \ + FIELD_GET(_bitmask, _rval); \ + }) + +/* Macro for writing a bitmask within a physically mapped packed struct */ +#define writel_relaxed_bitmask(_value, _regptr, _bitmask) \ + ({ \ + unsigned long _rval; \ + typeof(_regptr) _rptr = _regptr; \ + typeof(_bitmask) _bmask = _bitmask; \ + _rval = readl_relaxed(_rptr); \ + _rval &= ~(_bmask); \ + _rval |= FIELD_PREP((unsigned long long)_bmask, _value);\ + writel_relaxed(_rval, _rptr); \ + }) + +/* ====== MHUv3 data structures ====== */ + +enum mhuv3_frame { + PBX_FRAME, + MBX_FRAME, +}; + +static char *mhuv3_str[] = { + "PBX", + "MBX" +}; + +enum mhuv3_extension_type { + DBE_EXT, + FCE_EXT, + FE_EXT, + NUM_EXT +}; + +static char *mhuv3_ext_str[] = { + "DBE", + "FCE", + "FE" +}; + +struct mhuv3; + +/** + * struct mhuv3_protocol_ops - MHUv3 operations + * + * @rx_startup: Receiver startup callback. + * @rx_shutdown: Receiver shutdown callback. + * @read_data: Read available Sender in-band LE data (if any). + * @rx_complete: Acknowledge data reception to the Sender. Any out-of-band data + * has to have been already retrieved before calling this. + * @tx_startup: Sender startup callback. + * @tx_shutdown: Sender shutdown callback. + * @last_tx_done: Report back to the Sender if the last transfer has completed. + * @send_data: Send data to the receiver. + * + * Each supported transport protocol provides its own implementation of + * these operations. + */ +struct mhuv3_protocol_ops { + int (*rx_startup)(struct mhuv3 *mhu, struct mbox_chan *chan); + void (*rx_shutdown)(struct mhuv3 *mhu, struct mbox_chan *chan); + void *(*read_data)(struct mhuv3 *mhu, struct mbox_chan *chan); + void (*rx_complete)(struct mhuv3 *mhu, struct mbox_chan *chan); + void (*tx_startup)(struct mhuv3 *mhu, struct mbox_chan *chan); + void (*tx_shutdown)(struct mhuv3 *mhu, struct mbox_chan *chan); + int (*last_tx_done)(struct mhuv3 *mhu, struct mbox_chan *chan); + int (*send_data)(struct mhuv3 *mhu, struct mbox_chan *chan, void *arg); +}; + +/** + * struct mhuv3_mbox_chan_priv - MHUv3 channel private information + * + * @ch_idx: Channel window index associated to this mailbox channel. + * @doorbell: Doorbell bit number within the @ch_idx window. + * Only relevant to Doorbell transport. + * @ops: Transport protocol specific operations for this channel. + * + * Transport specific data attached to mmailbox channel priv data. + */ +struct mhuv3_mbox_chan_priv { + u32 ch_idx; + u32 doorbell; + const struct mhuv3_protocol_ops *ops; +}; + +/** + * struct mhuv3_extension - MHUv3 extension descriptor + * + * @type: Type of extension + * @num_chans: Max number of channels found for this extension. + * @base_ch_idx: First channel number assigned to this extension, picked from + * the set of all mailbox channels descriptors created. + * @mbox_of_xlate: Extension specific helper to parse DT and lookup associated + * channel from the related 'mboxes' property. + * @combined_irq_setup: Extension specific helper to setup the combined irq. + * @channels_init: Extension specific helper to initialize channels. + * @chan_from_comb_irq_get: Extension specific helper to lookup which channel + * triggered the combined irq. + * @pending_db: Array of per-channel pending doorbells. + * @pending_lock: Protect access to pending_db. + */ +struct mhuv3_extension { + enum mhuv3_extension_type type; + unsigned int num_chans; + unsigned int base_ch_idx; + struct mbox_chan *(*mbox_of_xlate)(struct mhuv3 *mhu, + unsigned int channel, + unsigned int param); + void (*combined_irq_setup)(struct mhuv3 *mhu); + int (*channels_init)(struct mhuv3 *mhu); + struct mbox_chan *(*chan_from_comb_irq_get)(struct mhuv3 *mhu); + u32 pending_db[MHUV3_DBCW_MAX]; + /* Protect access to pending_db */ + spinlock_t pending_lock; +}; + +/** + * struct mhuv3 - MHUv3 mailbox controller data + * + * @frame: Frame type: MBX_FRAME or PBX_FRAME. + * @auto_op_full: Flag to indicate if the MHU supports AutoOp full mode. + * @major: MHUv3 controller architectural major version. + * @minor: MHUv3 controller architectural minor version. + * @implem: MHUv3 controller IIDR implementer. + * @rev: MHUv3 controller IIDR revision. + * @var: MHUv3 controller IIDR variant. + * @prod_id: MHUv3 controller IIDR product_id. + * @num_chans: The total number of channnels discovered across all extensions. + * @cmb_irq: Combined IRQ number if any found defined. + * @ctrl: A reference to the MHUv3 control page for this block. + * @pbx: Base address of the PBX register mapping region. + * @mbx: Base address of the MBX register mapping region. + * @ext: Array holding descriptors for any found implemented extension. + * @mbox: Mailbox controller belonging to the MHU frame. + */ +struct mhuv3 { + enum mhuv3_frame frame; + bool auto_op_full; + unsigned int major; + unsigned int minor; + unsigned int implem; + unsigned int rev; + unsigned int var; + unsigned int prod_id; + unsigned int num_chans; + int cmb_irq; + struct ctrl_page __iomem *ctrl; + union { + struct mhu3_pbx_frame_reg __iomem *pbx; + struct mhu3_mbx_frame_reg __iomem *mbx; + }; + struct mhuv3_extension *ext[NUM_EXT]; + struct mbox_controller mbox; +}; + +#define mhu_from_mbox(_mbox) container_of(_mbox, struct mhuv3, mbox) + +typedef int (*mhuv3_extension_initializer)(struct mhuv3 *mhu); + +/* =================== Doorbell transport protocol operations =============== */ + +static void mhuv3_doorbell_tx_startup(struct mhuv3 *mhu, struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + + /* Enable Transfer Acknowledgment events */ + writel_relaxed_bitmask(0x1, &mhu->pbx->dbcw[priv->ch_idx].int_en, tfr_ack); +} + +static void mhuv3_doorbell_tx_shutdown(struct mhuv3 *mhu, struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + struct mhuv3_extension *e = mhu->ext[DBE_EXT]; + unsigned long flags; + + /* Disable Channel Transfer Ack events */ + writel_relaxed_bitmask(0x0, &mhu->pbx->dbcw[priv->ch_idx].int_en, tfr_ack); + + /* Clear Channel Transfer Ack and pending doorbells */ + writel_relaxed_bitmask(0x1, &mhu->pbx->dbcw[priv->ch_idx].int_clr, tfr_ack); + spin_lock_irqsave(&e->pending_lock, flags); + e->pending_db[priv->ch_idx] = 0; + spin_unlock_irqrestore(&e->pending_lock, flags); +} + +static int mhuv3_doorbell_rx_startup(struct mhuv3 *mhu, struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + + /* Unmask Channel Transfer events */ + writel_relaxed(BIT(priv->doorbell), &mhu->mbx->dbcw[priv->ch_idx].msk_clr); + + return 0; +} + +static void mhuv3_doorbell_rx_shutdown(struct mhuv3 *mhu, + struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + + /* Mask Channel Transfer events */ + writel_relaxed(BIT(priv->doorbell), &mhu->mbx->dbcw[priv->ch_idx].msk_set); +} + +static void mhuv3_doorbell_rx_complete(struct mhuv3 *mhu, struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + + /* Clearing the pending transfer generates the Channel Transfer Ack */ + writel_relaxed(BIT(priv->doorbell), &mhu->mbx->dbcw[priv->ch_idx].clr); +} + +static int mhuv3_doorbell_last_tx_done(struct mhuv3 *mhu, + struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + int done; + + done = !(readl_relaxed(&mhu->pbx->dbcw[priv->ch_idx].st) & + BIT(priv->doorbell)); + if (done) { + struct mhuv3_extension *e = mhu->ext[DBE_EXT]; + unsigned long flags; + + /* Take care to clear the pending doorbell also when polling */ + spin_lock_irqsave(&e->pending_lock, flags); + e->pending_db[priv->ch_idx] &= ~BIT(priv->doorbell); + spin_unlock_irqrestore(&e->pending_lock, flags); + } + + return done; +} + +static int mhuv3_doorbell_send_data(struct mhuv3 *mhu, struct mbox_chan *chan, + void *arg) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + struct mhuv3_extension *e = mhu->ext[DBE_EXT]; + + scoped_guard(spinlock_irqsave, &e->pending_lock) { + /* Only one in-flight Transfer is allowed per-doorbell */ + if (e->pending_db[priv->ch_idx] & BIT(priv->doorbell)) + return -EBUSY; + + e->pending_db[priv->ch_idx] |= BIT(priv->doorbell); + } + + writel_relaxed(BIT(priv->doorbell), &mhu->pbx->dbcw[priv->ch_idx].set); + + return 0; +} + +static const struct mhuv3_protocol_ops mhuv3_doorbell_ops = { + .tx_startup = mhuv3_doorbell_tx_startup, + .tx_shutdown = mhuv3_doorbell_tx_shutdown, + .rx_startup = mhuv3_doorbell_rx_startup, + .rx_shutdown = mhuv3_doorbell_rx_shutdown, + .rx_complete = mhuv3_doorbell_rx_complete, + .last_tx_done = mhuv3_doorbell_last_tx_done, + .send_data = mhuv3_doorbell_send_data, +}; + +/* Sender and receiver mailbox ops */ +static bool mhuv3_sender_last_tx_done(struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + struct mhuv3 *mhu = mhu_from_mbox(chan->mbox); + + return priv->ops->last_tx_done(mhu, chan); +} + +static int mhuv3_sender_send_data(struct mbox_chan *chan, void *data) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + struct mhuv3 *mhu = mhu_from_mbox(chan->mbox); + + if (!priv->ops->last_tx_done(mhu, chan)) + return -EBUSY; + + return priv->ops->send_data(mhu, chan, data); +} + +static int mhuv3_sender_startup(struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + struct mhuv3 *mhu = mhu_from_mbox(chan->mbox); + + if (priv->ops->tx_startup) + priv->ops->tx_startup(mhu, chan); + + return 0; +} + +static void mhuv3_sender_shutdown(struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + struct mhuv3 *mhu = mhu_from_mbox(chan->mbox); + + if (priv->ops->tx_shutdown) + priv->ops->tx_shutdown(mhu, chan); +} + +static const struct mbox_chan_ops mhuv3_sender_ops = { + .send_data = mhuv3_sender_send_data, + .startup = mhuv3_sender_startup, + .shutdown = mhuv3_sender_shutdown, + .last_tx_done = mhuv3_sender_last_tx_done, +}; + +static int mhuv3_receiver_startup(struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + struct mhuv3 *mhu = mhu_from_mbox(chan->mbox); + + return priv->ops->rx_startup(mhu, chan); +} + +static void mhuv3_receiver_shutdown(struct mbox_chan *chan) +{ + struct mhuv3_mbox_chan_priv *priv = chan->con_priv; + struct mhuv3 *mhu = mhu_from_mbox(chan->mbox); + + priv->ops->rx_shutdown(mhu, chan); +} + +static int mhuv3_receiver_send_data(struct mbox_chan *chan, void *data) +{ + dev_err(chan->mbox->dev, + "Trying to transmit on a MBX MHUv3 frame\n"); + return -EIO; +} + +static bool mhuv3_receiver_last_tx_done(struct mbox_chan *chan) +{ + dev_err(chan->mbox->dev, "Trying to Tx poll on a MBX MHUv3 frame\n"); + return true; +} + +static const struct mbox_chan_ops mhuv3_receiver_ops = { + .send_data = mhuv3_receiver_send_data, + .startup = mhuv3_receiver_startup, + .shutdown = mhuv3_receiver_shutdown, + .last_tx_done = mhuv3_receiver_last_tx_done, +}; + +static struct mbox_chan *mhuv3_dbe_mbox_of_xlate(struct mhuv3 *mhu, + unsigned int channel, + unsigned int doorbell) +{ + struct mhuv3_extension *e = mhu->ext[DBE_EXT]; + struct mbox_controller *mbox = &mhu->mbox; + struct mbox_chan *chans = mbox->chans; + + if (channel >= e->num_chans || doorbell >= MHUV3_FLAG_BITS) { + dev_err(mbox->dev, "Couldn't xlate to a valid channel (%d: %d)\n", + channel, doorbell); + return ERR_PTR(-ENODEV); + } + + return &chans[e->base_ch_idx + channel * MHUV3_FLAG_BITS + doorbell]; +} + +static void mhuv3_dbe_combined_irq_setup(struct mhuv3 *mhu) +{ + struct mhuv3_extension *e = mhu->ext[DBE_EXT]; + int i; + + if (mhu->frame == PBX_FRAME) { + struct pdbcw_page __iomem *dbcw = mhu->pbx->dbcw; + + for (i = 0; i < e->num_chans; i++) { + writel_relaxed_bitmask(0x1, &dbcw[i].int_clr, tfr_ack); + writel_relaxed_bitmask(0x0, &dbcw[i].int_en, tfr_ack); + writel_relaxed_bitmask(0x1, &dbcw[i].ctrl, comb_en); + } + } else { + struct mdbcw_page __iomem *dbcw = mhu->mbx->dbcw; + + for (i = 0; i < e->num_chans; i++) { + writel_relaxed(0xFFFFFFFF, &dbcw[i].clr); + writel_relaxed(0xFFFFFFFF, &dbcw[i].msk_set); + writel_relaxed_bitmask(0x1, &dbcw[i].ctrl, comb_en); + } + } +} + +static int mhuv3_dbe_channels_init(struct mhuv3 *mhu) +{ + struct mhuv3_extension *e = mhu->ext[DBE_EXT]; + struct mbox_controller *mbox = &mhu->mbox; + struct mbox_chan *chans; + int i; + + chans = mbox->chans + mbox->num_chans; + e->base_ch_idx = mbox->num_chans; + for (i = 0; i < e->num_chans; i++) { + struct mhuv3_mbox_chan_priv *priv; + int k; + + for (k = 0; k < MHUV3_FLAG_BITS; k++) { + priv = devm_kmalloc(mbox->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->ch_idx = i; + priv->ops = &mhuv3_doorbell_ops; + priv->doorbell = k; + chans++->con_priv = priv; + mbox->num_chans++; + } + } + + spin_lock_init(&e->pending_lock); + + return 0; +} + +static bool mhuv3_dbe_doorbell_lookup(struct mhuv3 *mhu, unsigned int channel, + unsigned int *db) +{ + struct mhuv3_extension *e = mhu->ext[DBE_EXT]; + struct device *dev = mhu->mbox.dev; + u32 st; + + if (mhu->frame == PBX_FRAME) { + u32 active_dbs, fired_dbs; + + st = readl_relaxed_bitmask(&mhu->pbx->dbcw[channel].int_st, + tfr_ack); + if (!st) + goto err_spurious; + + active_dbs = readl_relaxed(&mhu->pbx->dbcw[channel].st); + scoped_guard(spinlock_irqsave, &e->pending_lock) { + fired_dbs = e->pending_db[channel] & ~active_dbs; + if (!fired_dbs) + goto err_spurious; + + *db = __ffs(fired_dbs); + e->pending_db[channel] &= ~BIT(*db); + } + fired_dbs &= ~BIT(*db); + /* Clear TFR Ack if no more doorbells pending */ + if (!fired_dbs) + writel_relaxed_bitmask(0x1, + &mhu->pbx->dbcw[channel].int_clr, + tfr_ack); + } else { + st = readl_relaxed(&mhu->mbx->dbcw[channel].st_msk); + if (!st) + goto err_spurious; + + *db = __ffs(st); + } + + return true; + +err_spurious: + dev_warn(dev, "Spurious IRQ on %s channel:%d\n", + mhuv3_str[mhu->frame], channel); + + return false; +} + +static struct mbox_chan *mhuv3_dbe_chan_from_comb_irq_get(struct mhuv3 *mhu) +{ + struct mhuv3_extension *e = mhu->ext[DBE_EXT]; + struct device *dev = mhu->mbox.dev; + int i; + + for (i = 0; i < MHUV3_DBCH_CMB_INT_ST_REG_CNT; i++) { + unsigned int channel, db; + u32 cmb_st; + + cmb_st = readl_relaxed(&mhu->ctrl->dbch_int_st[i]); + if (!cmb_st) + continue; + + channel = i * MHUV3_FLAG_BITS + __ffs(cmb_st); + if (channel >= e->num_chans) { + dev_err(dev, "Invalid %s channel:%d\n", + mhuv3_str[mhu->frame], channel); + return ERR_PTR(-EIO); + } + + if (!mhuv3_dbe_doorbell_lookup(mhu, channel, &db)) + continue; + + dev_dbg(dev, "Found %s ch[%d]/db[%d]\n", + mhuv3_str[mhu->frame], channel, db); + + return &mhu->mbox.chans[channel * MHUV3_FLAG_BITS + db]; + } + + return ERR_PTR(-EIO); +} + +static int mhuv3_dbe_init(struct mhuv3 *mhu) +{ + struct device *dev = mhu->mbox.dev; + struct mhuv3_extension *e; + + if (!readl_relaxed_bitmask(&mhu->ctrl->feat_spt0, dbe_spt)) + return 0; + + dev_dbg(dev, "%s: Initializing DBE Extension.\n", mhuv3_str[mhu->frame]); + + e = devm_kzalloc(dev, sizeof(*e), GFP_KERNEL); + if (!e) + return -ENOMEM; + + e->type = DBE_EXT; + /* Note that, by the spec, the number of channels is (num_dbch + 1) */ + e->num_chans = + readl_relaxed_bitmask(&mhu->ctrl->dbch_cfg0, num_dbch) + 1; + e->mbox_of_xlate = mhuv3_dbe_mbox_of_xlate; + e->combined_irq_setup = mhuv3_dbe_combined_irq_setup; + e->channels_init = mhuv3_dbe_channels_init; + e->chan_from_comb_irq_get = mhuv3_dbe_chan_from_comb_irq_get; + + mhu->num_chans += e->num_chans * MHUV3_FLAG_BITS; + mhu->ext[DBE_EXT] = e; + + dev_dbg(dev, "%s: found %d DBE channels.\n", + mhuv3_str[mhu->frame], e->num_chans); + + return 0; +} + +static int mhuv3_fce_init(struct mhuv3 *mhu) +{ + struct device *dev = mhu->mbox.dev; + + if (!readl_relaxed_bitmask(&mhu->ctrl->feat_spt0, fce_spt)) + return 0; + + dev_dbg(dev, "%s: FCE Extension not supported by driver.\n", + mhuv3_str[mhu->frame]); + + return 0; +} + +static int mhuv3_fe_init(struct mhuv3 *mhu) +{ + struct device *dev = mhu->mbox.dev; + + if (!readl_relaxed_bitmask(&mhu->ctrl->feat_spt0, fe_spt)) + return 0; + + dev_dbg(dev, "%s: FE Extension not supported by driver.\n", + mhuv3_str[mhu->frame]); + + return 0; +} + +static mhuv3_extension_initializer mhuv3_extension_init[NUM_EXT] = { + mhuv3_dbe_init, + mhuv3_fce_init, + mhuv3_fe_init, +}; + +static int mhuv3_initialize_channels(struct device *dev, struct mhuv3 *mhu) +{ + struct mbox_controller *mbox = &mhu->mbox; + int i, ret = 0; + + mbox->chans = devm_kcalloc(dev, mhu->num_chans, + sizeof(*mbox->chans), GFP_KERNEL); + if (!mbox->chans) + return dev_err_probe(dev, -ENOMEM, + "Failed to initialize channels\n"); + + for (i = 0; i < NUM_EXT && !ret; i++) + if (mhu->ext[i]) + ret = mhu->ext[i]->channels_init(mhu); + + return ret; +} + +static struct mbox_chan *mhuv3_mbox_of_xlate(struct mbox_controller *mbox, + const struct of_phandle_args *pa) +{ + struct mhuv3 *mhu = mhu_from_mbox(mbox); + unsigned int type, channel, param; + + if (pa->args_count != MHUV3_MBOX_CELLS) + return ERR_PTR(-EINVAL); + + type = pa->args[MHUV3_MBOX_CELL_TYPE]; + if (type >= NUM_EXT) + return ERR_PTR(-EINVAL); + + channel = pa->args[MHUV3_MBOX_CELL_CHWN]; + param = pa->args[MHUV3_MBOX_CELL_PARAM]; + + return mhu->ext[type]->mbox_of_xlate(mhu, channel, param); +} + +static void mhu_frame_cleanup_actions(void *data) +{ + struct mhuv3 *mhu = data; + + writel_relaxed_bitmask(0x0, &mhu->ctrl->x_ctrl, op_req); +} + +static int mhuv3_frame_init(struct mhuv3 *mhu, void __iomem *regs) +{ + struct device *dev = mhu->mbox.dev; + int i; + + mhu->ctrl = regs; + mhu->frame = readl_relaxed_bitmask(&mhu->ctrl->blk_id, id); + if (mhu->frame > MBX_FRAME) + return dev_err_probe(dev, -EINVAL, + "Invalid Frame type- %d\n", mhu->frame); + + mhu->major = readl_relaxed_bitmask(&mhu->ctrl->aidr, arch_major_rev); + mhu->minor = readl_relaxed_bitmask(&mhu->ctrl->aidr, arch_minor_rev); + mhu->implem = readl_relaxed_bitmask(&mhu->ctrl->iidr, implementer); + mhu->rev = readl_relaxed_bitmask(&mhu->ctrl->iidr, revision); + mhu->var = readl_relaxed_bitmask(&mhu->ctrl->iidr, variant); + mhu->prod_id = readl_relaxed_bitmask(&mhu->ctrl->iidr, product_id); + if (mhu->major != MHUV3_MAJOR_VERSION) + return dev_err_probe(dev, -EINVAL, + "Unsupported MHU %s block - major:%d minor:%d\n", + mhuv3_str[mhu->frame], mhu->major, + mhu->minor); + + mhu->auto_op_full = + !!readl_relaxed_bitmask(&mhu->ctrl->feat_spt1, auto_op_spt); + /* Request the PBX/MBX to remain operational */ + if (mhu->auto_op_full) { + writel_relaxed_bitmask(0x1, &mhu->ctrl->x_ctrl, op_req); + devm_add_action_or_reset(dev, mhu_frame_cleanup_actions, mhu); + } + + dev_dbg(dev, + "Found MHU %s block - major:%d minor:%d\n implem:0x%X rev:0x%X var:0x%X prod_id:0x%X", + mhuv3_str[mhu->frame], mhu->major, mhu->minor, + mhu->implem, mhu->rev, mhu->var, mhu->prod_id); + + if (mhu->frame == PBX_FRAME) + mhu->pbx = regs; + else + mhu->mbx = regs; + + for (i = 0; i < NUM_EXT; i++) { + int ret; + + /* + * Note that extensions initialization fails only when such + * extension initialization routine fails and the extensions + * was found to be supported in hardware and in software. + */ + ret = mhuv3_extension_init[i](mhu); + if (ret) + return dev_err_probe(dev, ret, + "Failed to initialize %s %s\n", + mhuv3_str[mhu->frame], + mhuv3_ext_str[i]); + } + + return 0; +} + +static irqreturn_t mhuv3_pbx_comb_interrupt(int irq, void *arg) +{ + unsigned int i, found = 0; + struct mhuv3 *mhu = arg; + struct mbox_chan *chan; + struct device *dev; + int ret = IRQ_NONE; + + dev = mhu->mbox.dev; + for (i = 0; i < NUM_EXT; i++) { + struct mhuv3_mbox_chan_priv *priv; + + /* FCE does not participate to the PBX combined */ + if (i == FCE_EXT || !mhu->ext[i]) + continue; + + chan = mhu->ext[i]->chan_from_comb_irq_get(mhu); + if (IS_ERR(chan)) + continue; + + found++; + priv = chan->con_priv; + if (!chan->cl) { + dev_warn(dev, "TX Ack on UNBOUND channel (%u)\n", + priv->ch_idx); + continue; + } + + mbox_chan_txdone(chan, 0); + ret = IRQ_HANDLED; + } + + if (found == 0) + dev_warn_once(dev, "Failed to find channel for the TX interrupt\n"); + + return ret; +} + +static irqreturn_t mhuv3_mbx_comb_interrupt(int irq, void *arg) +{ + unsigned int i, found = 0; + struct mhuv3 *mhu = arg; + struct mbox_chan *chan; + struct device *dev; + int ret = IRQ_NONE; + + dev = mhu->mbox.dev; + for (i = 0; i < NUM_EXT; i++) { + struct mhuv3_mbox_chan_priv *priv; + void *data __free(kfree) = NULL; + + if (!mhu->ext[i]) + continue; + + /* Process any extension which could be source of the IRQ */ + chan = mhu->ext[i]->chan_from_comb_irq_get(mhu); + if (IS_ERR(chan)) + continue; + + found++; + /* From here on we need to call rx_complete even on error */ + priv = chan->con_priv; + if (!chan->cl) { + dev_warn(dev, "RX Data on UNBOUND channel (%u)\n", + priv->ch_idx); + goto rx_ack; + } + + /* Read optional in-band LE data first. */ + if (priv->ops->read_data) { + data = priv->ops->read_data(mhu, chan); + if (IS_ERR(data)) { + dev_err(dev, + "Failed to read in-band data. err:%ld\n", + PTR_ERR(no_free_ptr(data))); + goto rx_ack; + } + } + + mbox_chan_received_data(chan, data); + ret = IRQ_HANDLED; + + /* + * Acknowledge transfer after any possible optional + * out-of-band data has also been retrieved via + * mbox_chan_received_data(). + */ +rx_ack: + if (priv->ops->rx_complete) + priv->ops->rx_complete(mhu, chan); + } + + if (found == 0) + dev_warn_once(dev, "Failed to find channel for the RX interrupt\n"); + + return ret; +} + +static int mhuv3_setup_pbx(struct mhuv3 *mhu) +{ + struct device *dev = mhu->mbox.dev; + + mhu->mbox.ops = &mhuv3_sender_ops; + + if (mhu->cmb_irq > 0) { + int ret, i; + + ret = devm_request_threaded_irq(dev, mhu->cmb_irq, NULL, + mhuv3_pbx_comb_interrupt, + IRQF_ONESHOT, "mhuv3-pbx", mhu); + if (ret) + return dev_err_probe(dev, ret, + "Failed to request PBX IRQ\n"); + + mhu->mbox.txdone_irq = true; + mhu->mbox.txdone_poll = false; + + for (i = 0; i < NUM_EXT; i++) + if (mhu->ext[i]) + mhu->ext[i]->combined_irq_setup(mhu); + + dev_dbg(dev, "MHUv3 PBX IRQs initialized.\n"); + + return 0; + } + + dev_info(dev, "Using PBX in Tx polling mode.\n"); + mhu->mbox.txdone_irq = false; + mhu->mbox.txdone_poll = true; + mhu->mbox.txpoll_period = 1; + + return 0; +} + +static int mhuv3_setup_mbx(struct mhuv3 *mhu) +{ + struct device *dev = mhu->mbox.dev; + int ret, i; + + mhu->mbox.ops = &mhuv3_receiver_ops; + + if (mhu->cmb_irq <= 0) + return dev_err_probe(dev, -EINVAL, + "MBX combined IRQ is missing !\n"); + + ret = devm_request_threaded_irq(dev, mhu->cmb_irq, NULL, + mhuv3_mbx_comb_interrupt, IRQF_ONESHOT, + "mhuv3-mbx", mhu); + if (ret) + return dev_err_probe(dev, ret, "Failed to request MBX IRQ\n"); + + for (i = 0; i < NUM_EXT; i++) + if (mhu->ext[i]) + mhu->ext[i]->combined_irq_setup(mhu); + + dev_dbg(dev, "MHUv3 MBX IRQs initialized.\n"); + + return ret; +} + +static int mhuv3_irqs_init(struct mhuv3 *mhu, struct platform_device *pdev) +{ + dev_dbg(mhu->mbox.dev, "Initializing %s block.\n", + mhuv3_str[mhu->frame]); + + if (mhu->frame == PBX_FRAME) { + mhu->cmb_irq = + platform_get_irq_byname_optional(pdev, "combined"); + return mhuv3_setup_pbx(mhu); + } + + mhu->cmb_irq = platform_get_irq_byname(pdev, "combined"); + return mhuv3_setup_mbx(mhu); +} + +static int mhuv3_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + void __iomem *regs; + struct mhuv3 *mhu; + int ret; + + mhu = devm_kzalloc(dev, sizeof(*mhu), GFP_KERNEL); + if (!mhu) + return -ENOMEM; + + regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(regs)) + return PTR_ERR(regs); + + mhu->mbox.dev = dev; + ret = mhuv3_frame_init(mhu, regs); + if (ret) + return ret; + + ret = mhuv3_irqs_init(mhu, pdev); + if (ret) + return ret; + + mhu->mbox.of_xlate = mhuv3_mbox_of_xlate; + ret = mhuv3_initialize_channels(dev, mhu); + if (ret) + return ret; + + ret = devm_mbox_controller_register(dev, &mhu->mbox); + if (ret) + return dev_err_probe(dev, ret, + "Failed to register ARM MHUv3 driver\n"); + + return ret; +} + +static const struct of_device_id mhuv3_of_match[] = { + { .compatible = "arm,mhuv3", .data = NULL }, + {} +}; +MODULE_DEVICE_TABLE(of, mhuv3_of_match); + +static struct platform_driver mhuv3_driver = { + .driver = { + .name = "arm-mhuv3-mailbox", + .of_match_table = mhuv3_of_match, + }, + .probe = mhuv3_probe, +}; +module_platform_driver(mhuv3_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("ARM MHUv3 Driver"); +MODULE_AUTHOR("Cristian Marussi <cristian.marussi@arm.com>"); diff --git a/drivers/mailbox/bcm-pdc-mailbox.c b/drivers/mailbox/bcm-pdc-mailbox.c index 1768d3d5aaa0..242e7504a628 100644 --- a/drivers/mailbox/bcm-pdc-mailbox.c +++ b/drivers/mailbox/bcm-pdc-mailbox.c @@ -43,6 +43,7 @@ #include <linux/dma-direction.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> +#include <linux/workqueue.h> #define PDC_SUCCESS 0 @@ -293,8 +294,8 @@ struct pdc_state { unsigned int pdc_irq; - /* tasklet for deferred processing after DMA rx interrupt */ - struct tasklet_struct rx_tasklet; + /* work for deferred processing after DMA rx interrupt */ + struct work_struct rx_work; /* Number of bytes of receive status prior to each rx frame */ u32 rx_status_len; @@ -952,18 +953,18 @@ static irqreturn_t pdc_irq_handler(int irq, void *data) iowrite32(intstatus, pdcs->pdc_reg_vbase + PDC_INTSTATUS_OFFSET); /* Wakeup IRQ thread */ - tasklet_schedule(&pdcs->rx_tasklet); + queue_work(system_bh_wq, &pdcs->rx_work); return IRQ_HANDLED; } /** - * pdc_tasklet_cb() - Tasklet callback that runs the deferred processing after + * pdc_work_cb() - Work callback that runs the deferred processing after * a DMA receive interrupt. Reenables the receive interrupt. * @t: Pointer to the Altera sSGDMA channel structure */ -static void pdc_tasklet_cb(struct tasklet_struct *t) +static void pdc_work_cb(struct work_struct *t) { - struct pdc_state *pdcs = from_tasklet(pdcs, t, rx_tasklet); + struct pdc_state *pdcs = from_work(pdcs, t, rx_work); pdc_receive(pdcs); @@ -1577,8 +1578,8 @@ static int pdc_probe(struct platform_device *pdev) pdc_hw_init(pdcs); - /* Init tasklet for deferred DMA rx processing */ - tasklet_setup(&pdcs->rx_tasklet, pdc_tasklet_cb); + /* Init work for deferred DMA rx processing */ + INIT_WORK(&pdcs->rx_work, pdc_work_cb); err = pdc_interrupts_init(pdcs); if (err) @@ -1595,7 +1596,7 @@ static int pdc_probe(struct platform_device *pdev) return PDC_SUCCESS; cleanup_buf_pool: - tasklet_kill(&pdcs->rx_tasklet); + cancel_work_sync(&pdcs->rx_work); dma_pool_destroy(pdcs->rx_buf_pool); cleanup_ring_pool: @@ -1611,7 +1612,7 @@ static void pdc_remove(struct platform_device *pdev) pdc_free_debugfs(); - tasklet_kill(&pdcs->rx_tasklet); + cancel_work_sync(&pdcs->rx_work); pdc_hw_disable(pdcs); diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index 5c1d09cad761..933727f89431 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -21,6 +21,7 @@ #include <linux/pm_runtime.h> #include <linux/suspend.h> #include <linux/slab.h> +#include <linux/workqueue.h> #include "mailbox.h" @@ -80,7 +81,7 @@ struct imx_mu_con_priv { char irq_desc[IMX_MU_CHAN_NAME_SIZE]; enum imx_mu_chan_type type; struct mbox_chan *chan; - struct tasklet_struct txdb_tasklet; + struct work_struct txdb_work; }; struct imx_mu_priv { @@ -232,7 +233,7 @@ static int imx_mu_generic_tx(struct imx_mu_priv *priv, break; case IMX_MU_TYPE_TXDB: imx_mu_xcr_rmw(priv, IMX_MU_GCR, IMX_MU_xCR_GIRn(priv->dcfg->type, cp->idx), 0); - tasklet_schedule(&cp->txdb_tasklet); + queue_work(system_bh_wq, &cp->txdb_work); break; case IMX_MU_TYPE_TXDB_V2: imx_mu_xcr_rmw(priv, IMX_MU_GCR, IMX_MU_xCR_GIRn(priv->dcfg->type, cp->idx), 0); @@ -420,7 +421,7 @@ static int imx_mu_seco_tx(struct imx_mu_priv *priv, struct imx_mu_con_priv *cp, } /* Simulate hack for mbox framework */ - tasklet_schedule(&cp->txdb_tasklet); + queue_work(system_bh_wq, &cp->txdb_work); break; default: @@ -484,9 +485,9 @@ exit: return err; } -static void imx_mu_txdb_tasklet(unsigned long data) +static void imx_mu_txdb_work(struct work_struct *t) { - struct imx_mu_con_priv *cp = (struct imx_mu_con_priv *)data; + struct imx_mu_con_priv *cp = from_work(cp, t, txdb_work); mbox_chan_txdone(cp->chan, 0); } @@ -570,8 +571,7 @@ static int imx_mu_startup(struct mbox_chan *chan) if (cp->type == IMX_MU_TYPE_TXDB) { /* Tx doorbell don't have ACK support */ - tasklet_init(&cp->txdb_tasklet, imx_mu_txdb_tasklet, - (unsigned long)cp); + INIT_WORK(&cp->txdb_work, imx_mu_txdb_work); return 0; } @@ -615,7 +615,7 @@ static void imx_mu_shutdown(struct mbox_chan *chan) } if (cp->type == IMX_MU_TYPE_TXDB) { - tasklet_kill(&cp->txdb_tasklet); + cancel_work_sync(&cp->txdb_work); pm_runtime_put_sync(priv->dev); return; } diff --git a/drivers/mailbox/mtk-cmdq-mailbox.c b/drivers/mailbox/mtk-cmdq-mailbox.c index ead2200f39ba..4aa394e91109 100644 --- a/drivers/mailbox/mtk-cmdq-mailbox.c +++ b/drivers/mailbox/mtk-cmdq-mailbox.c @@ -465,7 +465,7 @@ static void cmdq_mbox_shutdown(struct mbox_chan *chan) struct cmdq_task *task, *tmp; unsigned long flags; - WARN_ON(pm_runtime_get_sync(cmdq->mbox.dev)); + WARN_ON(pm_runtime_get_sync(cmdq->mbox.dev) < 0); spin_lock_irqsave(&thread->chan->lock, flags); if (list_empty(&thread->task_busy_list)) @@ -765,6 +765,7 @@ static const struct of_device_id cmdq_of_ids[] = { {.compatible = "mediatek,mt8195-gce", .data = (void *)&gce_plat_mt8195}, {} }; +MODULE_DEVICE_TABLE(of, cmdq_of_ids); static struct platform_driver cmdq_drv = { .probe = cmdq_probe, diff --git a/drivers/mailbox/omap-mailbox.c b/drivers/mailbox/omap-mailbox.c index c961706fe61d..46747559b438 100644 --- a/drivers/mailbox/omap-mailbox.c +++ b/drivers/mailbox/omap-mailbox.c @@ -19,7 +19,6 @@ #include <linux/of.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> -#include <linux/omap-mailbox.h> #include <linux/mailbox_controller.h> #include <linux/mailbox_client.h> @@ -51,6 +50,11 @@ #define MBOX_INTR_CFG_TYPE1 0 #define MBOX_INTR_CFG_TYPE2 1 +typedef enum { + IRQ_TX = 1, + IRQ_RX = 2, +} omap_mbox_irq_t; + struct omap_mbox_fifo { unsigned long msg; unsigned long fifo_stat; @@ -61,14 +65,6 @@ struct omap_mbox_fifo { u32 intr_bit; }; -struct omap_mbox_queue { - spinlock_t lock; - struct kfifo fifo; - struct work_struct work; - struct omap_mbox *mbox; - bool full; -}; - struct omap_mbox_match_data { u32 intr_type; }; @@ -81,29 +77,11 @@ struct omap_mbox_device { u32 num_users; u32 num_fifos; u32 intr_type; - struct omap_mbox **mboxes; - struct mbox_controller controller; - struct list_head elem; -}; - -struct omap_mbox_fifo_info { - int tx_id; - int tx_usr; - int tx_irq; - - int rx_id; - int rx_usr; - int rx_irq; - - const char *name; - bool send_no_irq; }; struct omap_mbox { const char *name; int irq; - struct omap_mbox_queue *rxq; - struct device *dev; struct omap_mbox_device *parent; struct omap_mbox_fifo tx_fifo; struct omap_mbox_fifo rx_fifo; @@ -112,22 +90,6 @@ struct omap_mbox { bool send_no_irq; }; -/* global variables for the mailbox devices */ -static DEFINE_MUTEX(omap_mbox_devices_lock); -static LIST_HEAD(omap_mbox_devices); - -static unsigned int mbox_kfifo_size = CONFIG_OMAP_MBOX_KFIFO_SIZE; -module_param(mbox_kfifo_size, uint, S_IRUGO); -MODULE_PARM_DESC(mbox_kfifo_size, "Size of omap's mailbox kfifo (bytes)"); - -static struct omap_mbox *mbox_chan_to_omap_mbox(struct mbox_chan *chan) -{ - if (!chan || !chan->con_priv) - return NULL; - - return (struct omap_mbox *)chan->con_priv; -} - static inline unsigned int mbox_read_reg(struct omap_mbox_device *mdev, size_t ofs) { @@ -197,7 +159,7 @@ static int is_mbox_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq) return (int)(enable & status & bit); } -static void _omap_mbox_enable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq) +static void omap_mbox_enable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq) { u32 l; struct omap_mbox_fifo *fifo = (irq == IRQ_TX) ? @@ -210,7 +172,7 @@ static void _omap_mbox_enable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq) mbox_write_reg(mbox->parent, l, irqenable); } -static void _omap_mbox_disable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq) +static void omap_mbox_disable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq) { struct omap_mbox_fifo *fifo = (irq == IRQ_TX) ? &mbox->tx_fifo : &mbox->rx_fifo; @@ -227,87 +189,27 @@ static void _omap_mbox_disable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq) mbox_write_reg(mbox->parent, bit, irqdisable); } -void omap_mbox_enable_irq(struct mbox_chan *chan, omap_mbox_irq_t irq) -{ - struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan); - - if (WARN_ON(!mbox)) - return; - - _omap_mbox_enable_irq(mbox, irq); -} -EXPORT_SYMBOL(omap_mbox_enable_irq); - -void omap_mbox_disable_irq(struct mbox_chan *chan, omap_mbox_irq_t irq) -{ - struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan); - - if (WARN_ON(!mbox)) - return; - - _omap_mbox_disable_irq(mbox, irq); -} -EXPORT_SYMBOL(omap_mbox_disable_irq); - -/* - * Message receiver(workqueue) - */ -static void mbox_rx_work(struct work_struct *work) -{ - struct omap_mbox_queue *mq = - container_of(work, struct omap_mbox_queue, work); - mbox_msg_t data; - u32 msg; - int len; - - while (kfifo_len(&mq->fifo) >= sizeof(msg)) { - len = kfifo_out(&mq->fifo, (unsigned char *)&msg, sizeof(msg)); - WARN_ON(len != sizeof(msg)); - data = msg; - - mbox_chan_received_data(mq->mbox->chan, (void *)data); - spin_lock_irq(&mq->lock); - if (mq->full) { - mq->full = false; - _omap_mbox_enable_irq(mq->mbox, IRQ_RX); - } - spin_unlock_irq(&mq->lock); - } -} - /* * Mailbox interrupt handler */ static void __mbox_tx_interrupt(struct omap_mbox *mbox) { - _omap_mbox_disable_irq(mbox, IRQ_TX); + omap_mbox_disable_irq(mbox, IRQ_TX); ack_mbox_irq(mbox, IRQ_TX); mbox_chan_txdone(mbox->chan, 0); } static void __mbox_rx_interrupt(struct omap_mbox *mbox) { - struct omap_mbox_queue *mq = mbox->rxq; u32 msg; - int len; while (!mbox_fifo_empty(mbox)) { - if (unlikely(kfifo_avail(&mq->fifo) < sizeof(msg))) { - _omap_mbox_disable_irq(mbox, IRQ_RX); - mq->full = true; - goto nomem; - } - msg = mbox_fifo_read(mbox); - - len = kfifo_in(&mq->fifo, (unsigned char *)&msg, sizeof(msg)); - WARN_ON(len != sizeof(msg)); + mbox_chan_received_data(mbox->chan, (void *)(uintptr_t)msg); } - /* no more messages in the fifo. clear IRQ source. */ + /* clear IRQ source. */ ack_mbox_irq(mbox, IRQ_RX); -nomem: - schedule_work(&mbox->rxq->work); } static irqreturn_t mbox_interrupt(int irq, void *p) @@ -323,188 +225,34 @@ static irqreturn_t mbox_interrupt(int irq, void *p) return IRQ_HANDLED; } -static struct omap_mbox_queue *mbox_queue_alloc(struct omap_mbox *mbox, - void (*work)(struct work_struct *)) -{ - struct omap_mbox_queue *mq; - - if (!work) - return NULL; - - mq = kzalloc(sizeof(*mq), GFP_KERNEL); - if (!mq) - return NULL; - - spin_lock_init(&mq->lock); - - if (kfifo_alloc(&mq->fifo, mbox_kfifo_size, GFP_KERNEL)) - goto error; - - INIT_WORK(&mq->work, work); - return mq; - -error: - kfree(mq); - return NULL; -} - -static void mbox_queue_free(struct omap_mbox_queue *q) -{ - kfifo_free(&q->fifo); - kfree(q); -} - static int omap_mbox_startup(struct omap_mbox *mbox) { int ret = 0; - struct omap_mbox_queue *mq; - mq = mbox_queue_alloc(mbox, mbox_rx_work); - if (!mq) - return -ENOMEM; - mbox->rxq = mq; - mq->mbox = mbox; - - ret = request_irq(mbox->irq, mbox_interrupt, IRQF_SHARED, - mbox->name, mbox); + ret = request_threaded_irq(mbox->irq, NULL, mbox_interrupt, + IRQF_ONESHOT, mbox->name, mbox); if (unlikely(ret)) { pr_err("failed to register mailbox interrupt:%d\n", ret); - goto fail_request_irq; + return ret; } if (mbox->send_no_irq) mbox->chan->txdone_method = TXDONE_BY_ACK; - _omap_mbox_enable_irq(mbox, IRQ_RX); + omap_mbox_enable_irq(mbox, IRQ_RX); return 0; - -fail_request_irq: - mbox_queue_free(mbox->rxq); - return ret; } static void omap_mbox_fini(struct omap_mbox *mbox) { - _omap_mbox_disable_irq(mbox, IRQ_RX); + omap_mbox_disable_irq(mbox, IRQ_RX); free_irq(mbox->irq, mbox); - flush_work(&mbox->rxq->work); - mbox_queue_free(mbox->rxq); -} - -static struct omap_mbox *omap_mbox_device_find(struct omap_mbox_device *mdev, - const char *mbox_name) -{ - struct omap_mbox *_mbox, *mbox = NULL; - struct omap_mbox **mboxes = mdev->mboxes; - int i; - - if (!mboxes) - return NULL; - - for (i = 0; (_mbox = mboxes[i]); i++) { - if (!strcmp(_mbox->name, mbox_name)) { - mbox = _mbox; - break; - } - } - return mbox; -} - -struct mbox_chan *omap_mbox_request_channel(struct mbox_client *cl, - const char *chan_name) -{ - struct device *dev = cl->dev; - struct omap_mbox *mbox = NULL; - struct omap_mbox_device *mdev; - int ret; - - if (!dev) - return ERR_PTR(-ENODEV); - - if (dev->of_node) { - pr_err("%s: please use mbox_request_channel(), this API is supported only for OMAP non-DT usage\n", - __func__); - return ERR_PTR(-ENODEV); - } - - mutex_lock(&omap_mbox_devices_lock); - list_for_each_entry(mdev, &omap_mbox_devices, elem) { - mbox = omap_mbox_device_find(mdev, chan_name); - if (mbox) - break; - } - mutex_unlock(&omap_mbox_devices_lock); - - if (!mbox || !mbox->chan) - return ERR_PTR(-ENOENT); - - ret = mbox_bind_client(mbox->chan, cl); - if (ret) - return ERR_PTR(ret); - - return mbox->chan; -} -EXPORT_SYMBOL(omap_mbox_request_channel); - -static struct class omap_mbox_class = { .name = "mbox", }; - -static int omap_mbox_register(struct omap_mbox_device *mdev) -{ - int ret; - int i; - struct omap_mbox **mboxes; - - if (!mdev || !mdev->mboxes) - return -EINVAL; - - mboxes = mdev->mboxes; - for (i = 0; mboxes[i]; i++) { - struct omap_mbox *mbox = mboxes[i]; - - mbox->dev = device_create(&omap_mbox_class, mdev->dev, - 0, mbox, "%s", mbox->name); - if (IS_ERR(mbox->dev)) { - ret = PTR_ERR(mbox->dev); - goto err_out; - } - } - - mutex_lock(&omap_mbox_devices_lock); - list_add(&mdev->elem, &omap_mbox_devices); - mutex_unlock(&omap_mbox_devices_lock); - - ret = devm_mbox_controller_register(mdev->dev, &mdev->controller); - -err_out: - if (ret) { - while (i--) - device_unregister(mboxes[i]->dev); - } - return ret; -} - -static int omap_mbox_unregister(struct omap_mbox_device *mdev) -{ - int i; - struct omap_mbox **mboxes; - - if (!mdev || !mdev->mboxes) - return -EINVAL; - - mutex_lock(&omap_mbox_devices_lock); - list_del(&mdev->elem); - mutex_unlock(&omap_mbox_devices_lock); - - mboxes = mdev->mboxes; - for (i = 0; mboxes[i]; i++) - device_unregister(mboxes[i]->dev); - return 0; } static int omap_mbox_chan_startup(struct mbox_chan *chan) { - struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan); + struct omap_mbox *mbox = chan->con_priv; struct omap_mbox_device *mdev = mbox->parent; int ret = 0; @@ -519,7 +267,7 @@ static int omap_mbox_chan_startup(struct mbox_chan *chan) static void omap_mbox_chan_shutdown(struct mbox_chan *chan) { - struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan); + struct omap_mbox *mbox = chan->con_priv; struct omap_mbox_device *mdev = mbox->parent; mutex_lock(&mdev->cfg_lock); @@ -530,41 +278,40 @@ static void omap_mbox_chan_shutdown(struct mbox_chan *chan) static int omap_mbox_chan_send_noirq(struct omap_mbox *mbox, u32 msg) { - int ret = -EBUSY; + if (mbox_fifo_full(mbox)) + return -EBUSY; - if (!mbox_fifo_full(mbox)) { - _omap_mbox_enable_irq(mbox, IRQ_RX); - mbox_fifo_write(mbox, msg); - ret = 0; - _omap_mbox_disable_irq(mbox, IRQ_RX); + omap_mbox_enable_irq(mbox, IRQ_RX); + mbox_fifo_write(mbox, msg); + omap_mbox_disable_irq(mbox, IRQ_RX); - /* we must read and ack the interrupt directly from here */ - mbox_fifo_read(mbox); - ack_mbox_irq(mbox, IRQ_RX); - } + /* we must read and ack the interrupt directly from here */ + mbox_fifo_read(mbox); + ack_mbox_irq(mbox, IRQ_RX); - return ret; + return 0; } static int omap_mbox_chan_send(struct omap_mbox *mbox, u32 msg) { - int ret = -EBUSY; - - if (!mbox_fifo_full(mbox)) { - mbox_fifo_write(mbox, msg); - ret = 0; + if (mbox_fifo_full(mbox)) { + /* always enable the interrupt */ + omap_mbox_enable_irq(mbox, IRQ_TX); + return -EBUSY; } + mbox_fifo_write(mbox, msg); + /* always enable the interrupt */ - _omap_mbox_enable_irq(mbox, IRQ_TX); - return ret; + omap_mbox_enable_irq(mbox, IRQ_TX); + return 0; } static int omap_mbox_chan_send_data(struct mbox_chan *chan, void *data) { - struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan); + struct omap_mbox *mbox = chan->con_priv; int ret; - u32 msg = omap_mbox_message(data); + u32 msg = (u32)(uintptr_t)(data); if (!mbox) return -EINVAL; @@ -666,8 +413,9 @@ static struct mbox_chan *omap_mbox_of_xlate(struct mbox_controller *controller, struct device_node *node; struct omap_mbox_device *mdev; struct omap_mbox *mbox; + int i; - mdev = container_of(controller, struct omap_mbox_device, controller); + mdev = dev_get_drvdata(controller->dev); if (WARN_ON(!mdev)) return ERR_PTR(-EINVAL); @@ -678,22 +426,29 @@ static struct mbox_chan *omap_mbox_of_xlate(struct mbox_controller *controller, return ERR_PTR(-ENODEV); } - mbox = omap_mbox_device_find(mdev, node->name); + for (i = 0; i < controller->num_chans; i++) { + mbox = controller->chans[i].con_priv; + if (!strcmp(mbox->name, node->name)) { + of_node_put(node); + return &controller->chans[i]; + } + } + of_node_put(node); - return mbox ? mbox->chan : ERR_PTR(-ENOENT); + return ERR_PTR(-ENOENT); } static int omap_mbox_probe(struct platform_device *pdev) { int ret; struct mbox_chan *chnls; - struct omap_mbox **list, *mbox, *mboxblk; - struct omap_mbox_fifo_info *finfo, *finfoblk; + struct omap_mbox *mbox; struct omap_mbox_device *mdev; struct omap_mbox_fifo *fifo; struct device_node *node = pdev->dev.of_node; struct device_node *child; const struct omap_mbox_match_data *match_data; + struct mbox_controller *controller; u32 intr_type, info_count; u32 num_users, num_fifos; u32 tmp[3]; @@ -722,40 +477,6 @@ static int omap_mbox_probe(struct platform_device *pdev) return -ENODEV; } - finfoblk = devm_kcalloc(&pdev->dev, info_count, sizeof(*finfoblk), - GFP_KERNEL); - if (!finfoblk) - return -ENOMEM; - - finfo = finfoblk; - child = NULL; - for (i = 0; i < info_count; i++, finfo++) { - child = of_get_next_available_child(node, child); - ret = of_property_read_u32_array(child, "ti,mbox-tx", tmp, - ARRAY_SIZE(tmp)); - if (ret) - return ret; - finfo->tx_id = tmp[0]; - finfo->tx_irq = tmp[1]; - finfo->tx_usr = tmp[2]; - - ret = of_property_read_u32_array(child, "ti,mbox-rx", tmp, - ARRAY_SIZE(tmp)); - if (ret) - return ret; - finfo->rx_id = tmp[0]; - finfo->rx_irq = tmp[1]; - finfo->rx_usr = tmp[2]; - - finfo->name = child->name; - - finfo->send_no_irq = of_property_read_bool(child, "ti,mbox-send-noirq"); - - if (finfo->tx_id >= num_fifos || finfo->rx_id >= num_fifos || - finfo->tx_usr >= num_users || finfo->rx_usr >= num_users) - return -EINVAL; - } - mdev = devm_kzalloc(&pdev->dev, sizeof(*mdev), GFP_KERNEL); if (!mdev) return -ENOMEM; @@ -769,52 +490,67 @@ static int omap_mbox_probe(struct platform_device *pdev) if (!mdev->irq_ctx) return -ENOMEM; - /* allocate one extra for marking end of list */ - list = devm_kcalloc(&pdev->dev, info_count + 1, sizeof(*list), - GFP_KERNEL); - if (!list) - return -ENOMEM; - chnls = devm_kcalloc(&pdev->dev, info_count + 1, sizeof(*chnls), GFP_KERNEL); if (!chnls) return -ENOMEM; - mboxblk = devm_kcalloc(&pdev->dev, info_count, sizeof(*mbox), - GFP_KERNEL); - if (!mboxblk) - return -ENOMEM; + child = NULL; + for (i = 0; i < info_count; i++) { + int tx_id, tx_irq, tx_usr; + int rx_id, rx_usr; + + mbox = devm_kzalloc(&pdev->dev, sizeof(*mbox), GFP_KERNEL); + if (!mbox) + return -ENOMEM; + + child = of_get_next_available_child(node, child); + ret = of_property_read_u32_array(child, "ti,mbox-tx", tmp, + ARRAY_SIZE(tmp)); + if (ret) + return ret; + tx_id = tmp[0]; + tx_irq = tmp[1]; + tx_usr = tmp[2]; + + ret = of_property_read_u32_array(child, "ti,mbox-rx", tmp, + ARRAY_SIZE(tmp)); + if (ret) + return ret; + rx_id = tmp[0]; + /* rx_irq = tmp[1]; */ + rx_usr = tmp[2]; + + if (tx_id >= num_fifos || rx_id >= num_fifos || + tx_usr >= num_users || rx_usr >= num_users) + return -EINVAL; - mbox = mboxblk; - finfo = finfoblk; - for (i = 0; i < info_count; i++, finfo++) { fifo = &mbox->tx_fifo; - fifo->msg = MAILBOX_MESSAGE(finfo->tx_id); - fifo->fifo_stat = MAILBOX_FIFOSTATUS(finfo->tx_id); - fifo->intr_bit = MAILBOX_IRQ_NOTFULL(finfo->tx_id); - fifo->irqenable = MAILBOX_IRQENABLE(intr_type, finfo->tx_usr); - fifo->irqstatus = MAILBOX_IRQSTATUS(intr_type, finfo->tx_usr); - fifo->irqdisable = MAILBOX_IRQDISABLE(intr_type, finfo->tx_usr); + fifo->msg = MAILBOX_MESSAGE(tx_id); + fifo->fifo_stat = MAILBOX_FIFOSTATUS(tx_id); + fifo->intr_bit = MAILBOX_IRQ_NOTFULL(tx_id); + fifo->irqenable = MAILBOX_IRQENABLE(intr_type, tx_usr); + fifo->irqstatus = MAILBOX_IRQSTATUS(intr_type, tx_usr); + fifo->irqdisable = MAILBOX_IRQDISABLE(intr_type, tx_usr); fifo = &mbox->rx_fifo; - fifo->msg = MAILBOX_MESSAGE(finfo->rx_id); - fifo->msg_stat = MAILBOX_MSGSTATUS(finfo->rx_id); - fifo->intr_bit = MAILBOX_IRQ_NEWMSG(finfo->rx_id); - fifo->irqenable = MAILBOX_IRQENABLE(intr_type, finfo->rx_usr); - fifo->irqstatus = MAILBOX_IRQSTATUS(intr_type, finfo->rx_usr); - fifo->irqdisable = MAILBOX_IRQDISABLE(intr_type, finfo->rx_usr); - - mbox->send_no_irq = finfo->send_no_irq; + fifo->msg = MAILBOX_MESSAGE(rx_id); + fifo->msg_stat = MAILBOX_MSGSTATUS(rx_id); + fifo->intr_bit = MAILBOX_IRQ_NEWMSG(rx_id); + fifo->irqenable = MAILBOX_IRQENABLE(intr_type, rx_usr); + fifo->irqstatus = MAILBOX_IRQSTATUS(intr_type, rx_usr); + fifo->irqdisable = MAILBOX_IRQDISABLE(intr_type, rx_usr); + + mbox->send_no_irq = of_property_read_bool(child, "ti,mbox-send-noirq"); mbox->intr_type = intr_type; mbox->parent = mdev; - mbox->name = finfo->name; - mbox->irq = platform_get_irq(pdev, finfo->tx_irq); + mbox->name = child->name; + mbox->irq = platform_get_irq(pdev, tx_irq); if (mbox->irq < 0) return mbox->irq; mbox->chan = &chnls[i]; chnls[i].con_priv = mbox; - list[i] = mbox++; } mutex_init(&mdev->cfg_lock); @@ -822,28 +558,30 @@ static int omap_mbox_probe(struct platform_device *pdev) mdev->num_users = num_users; mdev->num_fifos = num_fifos; mdev->intr_type = intr_type; - mdev->mboxes = list; + controller = devm_kzalloc(&pdev->dev, sizeof(*controller), GFP_KERNEL); + if (!controller) + return -ENOMEM; /* * OMAP/K3 Mailbox IP does not have a Tx-Done IRQ, but rather a Tx-Ready * IRQ and is needed to run the Tx state machine */ - mdev->controller.txdone_irq = true; - mdev->controller.dev = mdev->dev; - mdev->controller.ops = &omap_mbox_chan_ops; - mdev->controller.chans = chnls; - mdev->controller.num_chans = info_count; - mdev->controller.of_xlate = omap_mbox_of_xlate; - ret = omap_mbox_register(mdev); + controller->txdone_irq = true; + controller->dev = mdev->dev; + controller->ops = &omap_mbox_chan_ops; + controller->chans = chnls; + controller->num_chans = info_count; + controller->of_xlate = omap_mbox_of_xlate; + ret = devm_mbox_controller_register(mdev->dev, controller); if (ret) return ret; platform_set_drvdata(pdev, mdev); - pm_runtime_enable(mdev->dev); + devm_pm_runtime_enable(mdev->dev); ret = pm_runtime_resume_and_get(mdev->dev); if (ret < 0) - goto unregister; + return ret; /* * just print the raw revision register, the format is not @@ -854,61 +592,20 @@ static int omap_mbox_probe(struct platform_device *pdev) ret = pm_runtime_put_sync(mdev->dev); if (ret < 0 && ret != -ENOSYS) - goto unregister; + return ret; - devm_kfree(&pdev->dev, finfoblk); return 0; - -unregister: - pm_runtime_disable(mdev->dev); - omap_mbox_unregister(mdev); - return ret; -} - -static void omap_mbox_remove(struct platform_device *pdev) -{ - struct omap_mbox_device *mdev = platform_get_drvdata(pdev); - - pm_runtime_disable(mdev->dev); - omap_mbox_unregister(mdev); } static struct platform_driver omap_mbox_driver = { .probe = omap_mbox_probe, - .remove_new = omap_mbox_remove, .driver = { .name = "omap-mailbox", .pm = &omap_mbox_pm_ops, .of_match_table = of_match_ptr(omap_mailbox_of_match), }, }; - -static int __init omap_mbox_init(void) -{ - int err; - - err = class_register(&omap_mbox_class); - if (err) - return err; - - /* kfifo size sanity check: alignment and minimal size */ - mbox_kfifo_size = ALIGN(mbox_kfifo_size, sizeof(u32)); - mbox_kfifo_size = max_t(unsigned int, mbox_kfifo_size, sizeof(u32)); - - err = platform_driver_register(&omap_mbox_driver); - if (err) - class_unregister(&omap_mbox_class); - - return err; -} -subsys_initcall(omap_mbox_init); - -static void __exit omap_mbox_exit(void) -{ - platform_driver_unregister(&omap_mbox_driver); - class_unregister(&omap_mbox_class); -} -module_exit(omap_mbox_exit); +module_platform_driver(omap_mbox_driver); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("omap mailbox: interrupt driven messaging"); diff --git a/drivers/mailbox/zynqmp-ipi-mailbox.c b/drivers/mailbox/zynqmp-ipi-mailbox.c index 25c65afc030a..7c90bac3de21 100644 --- a/drivers/mailbox/zynqmp-ipi-mailbox.c +++ b/drivers/mailbox/zynqmp-ipi-mailbox.c @@ -6,9 +6,11 @@ */ #include <linux/arm-smccc.h> +#include <linux/cpuhotplug.h> #include <linux/delay.h> #include <linux/device.h> #include <linux/interrupt.h> +#include <linux/irqdomain.h> #include <linux/io.h> #include <linux/kernel.h> #include <linux/mailbox_controller.h> @@ -16,6 +18,7 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/of_address.h> +#include <linux/of_irq.h> #include <linux/platform_device.h> /* IPI agent ID any */ @@ -52,6 +55,15 @@ #define IPI_MB_CHNL_TX 0 /* IPI mailbox TX channel */ #define IPI_MB_CHNL_RX 1 /* IPI mailbox RX channel */ +/* IPI Message Buffer Information */ +#define RESP_OFFSET 0x20U +#define DEST_OFFSET 0x40U +#define IPI_BUF_SIZE 0x20U +#define DST_BIT_POS 9U +#define SRC_BITMASK GENMASK(11, 8) + +#define MAX_SGI 16 + /** * struct zynqmp_ipi_mchan - Description of a Xilinx ZynqMP IPI mailbox channel * @is_opened: indicate if the IPI channel is opened @@ -72,6 +84,10 @@ struct zynqmp_ipi_mchan { unsigned int chan_type; }; +struct zynqmp_ipi_mbox; + +typedef int (*setup_ipi_fn)(struct zynqmp_ipi_mbox *ipi_mbox, struct device_node *node); + /** * struct zynqmp_ipi_mbox - Description of a ZynqMP IPI mailbox * platform data. @@ -81,6 +97,7 @@ struct zynqmp_ipi_mchan { * @remote_id: remote IPI agent ID * @mbox: mailbox Controller * @mchans: array for channels, tx channel and rx channel. + * @setup_ipi_fn: Function Pointer to set up IPI Channels */ struct zynqmp_ipi_mbox { struct zynqmp_ipi_pdata *pdata; @@ -88,6 +105,7 @@ struct zynqmp_ipi_mbox { u32 remote_id; struct mbox_controller mbox; struct zynqmp_ipi_mchan mchans[2]; + setup_ipi_fn setup_ipi_fn; }; /** @@ -98,6 +116,7 @@ struct zynqmp_ipi_mbox { * @irq: IPI agent interrupt ID * @method: IPI SMC or HVC is going to be used * @local_id: local IPI agent ID + * @virq_sgi: IRQ number mapped to SGI * @num_mboxes: number of mailboxes of this IPI agent * @ipi_mboxes: IPI mailboxes of this IPI agent */ @@ -106,10 +125,13 @@ struct zynqmp_ipi_pdata { int irq; unsigned int method; u32 local_id; + int virq_sgi; int num_mboxes; struct zynqmp_ipi_mbox ipi_mboxes[] __counted_by(num_mboxes); }; +static DEFINE_PER_CPU(struct zynqmp_ipi_pdata *, per_cpu_pdata); + static struct device_driver zynqmp_ipi_mbox_driver = { .owner = THIS_MODULE, .name = "zynqmp-ipi-mbox", @@ -163,9 +185,11 @@ static irqreturn_t zynqmp_ipi_interrupt(int irq, void *data) if (ret > 0 && ret & IPI_MB_STATUS_RECV_PENDING) { if (mchan->is_opened) { msg = mchan->rx_buf; - msg->len = mchan->req_buf_size; - memcpy_fromio(msg->data, mchan->req_buf, - msg->len); + if (msg) { + msg->len = mchan->req_buf_size; + memcpy_fromio(msg->data, mchan->req_buf, + msg->len); + } mbox_chan_received_data(chan, (void *)msg); status = IRQ_HANDLED; } @@ -174,6 +198,14 @@ static irqreturn_t zynqmp_ipi_interrupt(int irq, void *data) return status; } +static irqreturn_t zynqmp_sgi_interrupt(int irq, void *data) +{ + struct zynqmp_ipi_pdata **pdata_ptr = data; + struct zynqmp_ipi_pdata *pdata = *pdata_ptr; + + return zynqmp_ipi_interrupt(irq, pdata); +} + /** * zynqmp_ipi_peek_data - Peek to see if there are any rx messages. * @@ -275,26 +307,26 @@ static int zynqmp_ipi_send_data(struct mbox_chan *chan, void *data) if (mchan->chan_type == IPI_MB_CHNL_TX) { /* Send request message */ - if (msg && msg->len > mchan->req_buf_size) { + if (msg && msg->len > mchan->req_buf_size && mchan->req_buf) { dev_err(dev, "channel %d message length %u > max %lu\n", mchan->chan_type, (unsigned int)msg->len, mchan->req_buf_size); return -EINVAL; } - if (msg && msg->len) + if (msg && msg->len && mchan->req_buf) memcpy_toio(mchan->req_buf, msg->data, msg->len); /* Kick IPI mailbox to send message */ arg0 = SMC_IPI_MAILBOX_NOTIFY; zynqmp_ipi_fw_call(ipi_mbox, arg0, 0, &res); } else { /* Send response message */ - if (msg && msg->len > mchan->resp_buf_size) { + if (msg && msg->len > mchan->resp_buf_size && mchan->resp_buf) { dev_err(dev, "channel %d message length %u > max %lu\n", mchan->chan_type, (unsigned int)msg->len, mchan->resp_buf_size); return -EINVAL; } - if (msg && msg->len) + if (msg && msg->len && mchan->resp_buf) memcpy_toio(mchan->resp_buf, msg->data, msg->len); arg0 = SMC_IPI_MAILBOX_ACK; zynqmp_ipi_fw_call(ipi_mbox, arg0, IPI_SMC_ACK_EIRQ_MASK, @@ -415,12 +447,6 @@ static struct mbox_chan *zynqmp_ipi_of_xlate(struct mbox_controller *mbox, return chan; } -static const struct of_device_id zynqmp_ipi_of_match[] = { - { .compatible = "xlnx,zynqmp-ipi-mailbox" }, - {}, -}; -MODULE_DEVICE_TABLE(of, zynqmp_ipi_of_match); - /** * zynqmp_ipi_mbox_get_buf_res - Get buffer resource from the IPI dev node * @@ -470,12 +496,9 @@ static void zynqmp_ipi_mbox_dev_release(struct device *dev) static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox, struct device_node *node) { - struct zynqmp_ipi_mchan *mchan; struct mbox_chan *chans; struct mbox_controller *mbox; - struct resource res; struct device *dev, *mdev; - const char *name; int ret; dev = ipi_mbox->pdata->dev; @@ -495,6 +518,73 @@ static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox, } mdev = &ipi_mbox->dev; + /* Get the IPI remote agent ID */ + ret = of_property_read_u32(node, "xlnx,ipi-id", &ipi_mbox->remote_id); + if (ret < 0) { + dev_err(dev, "No IPI remote ID is specified.\n"); + return ret; + } + + ret = ipi_mbox->setup_ipi_fn(ipi_mbox, node); + if (ret) { + dev_err(dev, "Failed to set up IPI Buffers.\n"); + return ret; + } + + mbox = &ipi_mbox->mbox; + mbox->dev = mdev; + mbox->ops = &zynqmp_ipi_chan_ops; + mbox->num_chans = 2; + mbox->txdone_irq = false; + mbox->txdone_poll = true; + mbox->txpoll_period = 5; + mbox->of_xlate = zynqmp_ipi_of_xlate; + chans = devm_kzalloc(mdev, 2 * sizeof(*chans), GFP_KERNEL); + if (!chans) + return -ENOMEM; + mbox->chans = chans; + chans[IPI_MB_CHNL_TX].con_priv = &ipi_mbox->mchans[IPI_MB_CHNL_TX]; + chans[IPI_MB_CHNL_RX].con_priv = &ipi_mbox->mchans[IPI_MB_CHNL_RX]; + ipi_mbox->mchans[IPI_MB_CHNL_TX].chan_type = IPI_MB_CHNL_TX; + ipi_mbox->mchans[IPI_MB_CHNL_RX].chan_type = IPI_MB_CHNL_RX; + ret = devm_mbox_controller_register(mdev, mbox); + if (ret) + dev_err(mdev, + "Failed to register mbox_controller(%d)\n", ret); + else + dev_info(mdev, + "Registered ZynqMP IPI mbox with TX/RX channels.\n"); + return ret; +} + +/** + * zynqmp_ipi_setup - set up IPI Buffers for classic flow + * + * @ipi_mbox: pointer to IPI mailbox private data structure + * @node: IPI mailbox device node + * + * This will be used to set up IPI Buffers for ZynqMP SOC if user + * wishes to use classic driver usage model on new SOC's with only + * buffered IPIs. + * + * Note that bufferless IPIs and mixed usage of buffered and bufferless + * IPIs are not supported with this flow. + * + * This will be invoked with compatible string "xlnx,zynqmp-ipi-mailbox". + * + * Return: 0 for success, negative value for failure + */ +static int zynqmp_ipi_setup(struct zynqmp_ipi_mbox *ipi_mbox, + struct device_node *node) +{ + struct zynqmp_ipi_mchan *mchan; + struct device *mdev; + struct resource res; + const char *name; + int ret; + + mdev = &ipi_mbox->dev; + mchan = &ipi_mbox->mchans[IPI_MB_CHNL_TX]; name = "local_request_region"; ret = zynqmp_ipi_mbox_get_buf_res(node, name, &res); @@ -569,39 +659,218 @@ static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox, if (!mchan->rx_buf) return -ENOMEM; - /* Get the IPI remote agent ID */ - ret = of_property_read_u32(node, "xlnx,ipi-id", &ipi_mbox->remote_id); - if (ret < 0) { - dev_err(dev, "No IPI remote ID is specified.\n"); + return 0; +} + +/** + * versal_ipi_setup - Set up IPIs to support mixed usage of + * Buffered and Bufferless IPIs. + * + * @ipi_mbox: pointer to IPI mailbox private data structure + * @node: IPI mailbox device node + * + * Return: 0 for success, negative value for failure + */ +static int versal_ipi_setup(struct zynqmp_ipi_mbox *ipi_mbox, + struct device_node *node) +{ + struct zynqmp_ipi_mchan *tx_mchan, *rx_mchan; + struct resource host_res, remote_res; + struct device_node *parent_node; + int host_idx, remote_idx; + struct device *mdev; + + tx_mchan = &ipi_mbox->mchans[IPI_MB_CHNL_TX]; + rx_mchan = &ipi_mbox->mchans[IPI_MB_CHNL_RX]; + parent_node = of_get_parent(node); + mdev = &ipi_mbox->dev; + + host_idx = zynqmp_ipi_mbox_get_buf_res(parent_node, "msg", &host_res); + remote_idx = zynqmp_ipi_mbox_get_buf_res(node, "msg", &remote_res); + + /* + * Only set up buffers if both sides claim to have msg buffers. + * This is because each buffered IPI's corresponding msg buffers + * are reserved for use by other buffered IPI's. + */ + if (!host_idx && !remote_idx) { + u32 host_src, host_dst, remote_src, remote_dst; + u32 buff_sz; + + buff_sz = resource_size(&host_res); + + host_src = host_res.start & SRC_BITMASK; + remote_src = remote_res.start & SRC_BITMASK; + + host_dst = (host_src >> DST_BIT_POS) * DEST_OFFSET; + remote_dst = (remote_src >> DST_BIT_POS) * DEST_OFFSET; + + /* Validate that IPI IDs is within IPI Message buffer space. */ + if (host_dst >= buff_sz || remote_dst >= buff_sz) { + dev_err(mdev, + "Invalid IPI Message buffer values: %x %x\n", + host_dst, remote_dst); + return -EINVAL; + } + + tx_mchan->req_buf = devm_ioremap(mdev, + host_res.start | remote_dst, + IPI_BUF_SIZE); + if (!tx_mchan->req_buf) { + dev_err(mdev, "Unable to map IPI buffer I/O memory\n"); + return -ENOMEM; + } + + tx_mchan->resp_buf = devm_ioremap(mdev, + (remote_res.start | host_dst) + + RESP_OFFSET, IPI_BUF_SIZE); + if (!tx_mchan->resp_buf) { + dev_err(mdev, "Unable to map IPI buffer I/O memory\n"); + return -ENOMEM; + } + + rx_mchan->req_buf = devm_ioremap(mdev, + remote_res.start | host_dst, + IPI_BUF_SIZE); + if (!rx_mchan->req_buf) { + dev_err(mdev, "Unable to map IPI buffer I/O memory\n"); + return -ENOMEM; + } + + rx_mchan->resp_buf = devm_ioremap(mdev, + (host_res.start | remote_dst) + + RESP_OFFSET, IPI_BUF_SIZE); + if (!rx_mchan->resp_buf) { + dev_err(mdev, "Unable to map IPI buffer I/O memory\n"); + return -ENOMEM; + } + + tx_mchan->resp_buf_size = IPI_BUF_SIZE; + tx_mchan->req_buf_size = IPI_BUF_SIZE; + tx_mchan->rx_buf = devm_kzalloc(mdev, IPI_BUF_SIZE + + sizeof(struct zynqmp_ipi_message), + GFP_KERNEL); + if (!tx_mchan->rx_buf) + return -ENOMEM; + + rx_mchan->resp_buf_size = IPI_BUF_SIZE; + rx_mchan->req_buf_size = IPI_BUF_SIZE; + rx_mchan->rx_buf = devm_kzalloc(mdev, IPI_BUF_SIZE + + sizeof(struct zynqmp_ipi_message), + GFP_KERNEL); + if (!rx_mchan->rx_buf) + return -ENOMEM; + } + + return 0; +} + +static int xlnx_mbox_cpuhp_start(unsigned int cpu) +{ + struct zynqmp_ipi_pdata *pdata; + + pdata = get_cpu_var(per_cpu_pdata); + put_cpu_var(per_cpu_pdata); + enable_percpu_irq(pdata->virq_sgi, IRQ_TYPE_NONE); + + return 0; +} + +static int xlnx_mbox_cpuhp_down(unsigned int cpu) +{ + struct zynqmp_ipi_pdata *pdata; + + pdata = get_cpu_var(per_cpu_pdata); + put_cpu_var(per_cpu_pdata); + disable_percpu_irq(pdata->virq_sgi); + + return 0; +} + +static void xlnx_disable_percpu_irq(void *data) +{ + struct zynqmp_ipi_pdata *pdata; + + pdata = *this_cpu_ptr(&per_cpu_pdata); + + disable_percpu_irq(pdata->virq_sgi); +} + +static int xlnx_mbox_init_sgi(struct platform_device *pdev, + int sgi_num, + struct zynqmp_ipi_pdata *pdata) +{ + int ret = 0; + int cpu; + + /* + * IRQ related structures are used for the following: + * for each SGI interrupt ensure its mapped by GIC IRQ domain + * and that each corresponding linux IRQ for the HW IRQ has + * a handler for when receiving an interrupt from the remote + * processor. + */ + struct irq_domain *domain; + struct irq_fwspec sgi_fwspec; + struct device_node *interrupt_parent = NULL; + struct device *dev = &pdev->dev; + + /* Find GIC controller to map SGIs. */ + interrupt_parent = of_irq_find_parent(dev->of_node); + if (!interrupt_parent) { + dev_err(&pdev->dev, "Failed to find property for Interrupt parent\n"); + return -EINVAL; + } + + /* Each SGI needs to be associated with GIC's IRQ domain. */ + domain = irq_find_host(interrupt_parent); + of_node_put(interrupt_parent); + + /* Each mapping needs GIC domain when finding IRQ mapping. */ + sgi_fwspec.fwnode = domain->fwnode; + + /* + * When irq domain looks at mapping each arg is as follows: + * 3 args for: interrupt type (SGI), interrupt # (set later), type + */ + sgi_fwspec.param_count = 1; + + /* Set SGI's hwirq */ + sgi_fwspec.param[0] = sgi_num; + pdata->virq_sgi = irq_create_fwspec_mapping(&sgi_fwspec); + + for_each_possible_cpu(cpu) + per_cpu(per_cpu_pdata, cpu) = pdata; + + ret = request_percpu_irq(pdata->virq_sgi, zynqmp_sgi_interrupt, pdev->name, + &per_cpu_pdata); + WARN_ON(ret); + if (ret) { + irq_dispose_mapping(pdata->virq_sgi); return ret; } - mbox = &ipi_mbox->mbox; - mbox->dev = mdev; - mbox->ops = &zynqmp_ipi_chan_ops; - mbox->num_chans = 2; - mbox->txdone_irq = false; - mbox->txdone_poll = true; - mbox->txpoll_period = 5; - mbox->of_xlate = zynqmp_ipi_of_xlate; - chans = devm_kzalloc(mdev, 2 * sizeof(*chans), GFP_KERNEL); - if (!chans) - return -ENOMEM; - mbox->chans = chans; - chans[IPI_MB_CHNL_TX].con_priv = &ipi_mbox->mchans[IPI_MB_CHNL_TX]; - chans[IPI_MB_CHNL_RX].con_priv = &ipi_mbox->mchans[IPI_MB_CHNL_RX]; - ipi_mbox->mchans[IPI_MB_CHNL_TX].chan_type = IPI_MB_CHNL_TX; - ipi_mbox->mchans[IPI_MB_CHNL_RX].chan_type = IPI_MB_CHNL_RX; - ret = devm_mbox_controller_register(mdev, mbox); - if (ret) - dev_err(mdev, - "Failed to register mbox_controller(%d)\n", ret); - else - dev_info(mdev, - "Registered ZynqMP IPI mbox with TX/RX channels.\n"); + irq_to_desc(pdata->virq_sgi); + irq_set_status_flags(pdata->virq_sgi, IRQ_PER_CPU); + + /* Setup function for the CPU hot-plug cases */ + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mailbox/sgi:starting", + xlnx_mbox_cpuhp_start, xlnx_mbox_cpuhp_down); + return ret; } +static void xlnx_mbox_cleanup_sgi(struct zynqmp_ipi_pdata *pdata) +{ + cpuhp_remove_state(CPUHP_AP_ONLINE_DYN); + + on_each_cpu(xlnx_disable_percpu_irq, NULL, 1); + + irq_clear_status_flags(pdata->virq_sgi, IRQ_PER_CPU); + free_percpu_irq(pdata->virq_sgi, &per_cpu_pdata); + irq_dispose_mapping(pdata->virq_sgi); +} + /** * zynqmp_ipi_free_mboxes - Free IPI mailboxes devices * @@ -612,6 +881,9 @@ static void zynqmp_ipi_free_mboxes(struct zynqmp_ipi_pdata *pdata) struct zynqmp_ipi_mbox *ipi_mbox; int i; + if (pdata->irq < MAX_SGI) + xlnx_mbox_cleanup_sgi(pdata); + i = pdata->num_mboxes; for (; i >= 0; i--) { ipi_mbox = &pdata->ipi_mboxes[i]; @@ -627,9 +899,11 @@ static int zynqmp_ipi_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *nc, *np = pdev->dev.of_node; - struct zynqmp_ipi_pdata *pdata; + struct zynqmp_ipi_pdata __percpu *pdata; + struct of_phandle_args out_irq; struct zynqmp_ipi_mbox *mbox; int num_mboxes, ret = -EINVAL; + setup_ipi_fn ipi_fn; num_mboxes = of_get_available_child_count(np); if (num_mboxes == 0) { @@ -650,9 +924,18 @@ static int zynqmp_ipi_probe(struct platform_device *pdev) return ret; } + ipi_fn = (setup_ipi_fn)device_get_match_data(&pdev->dev); + if (!ipi_fn) { + dev_err(dev, + "Mbox Compatible String is missing IPI Setup fn.\n"); + return -ENODEV; + } + pdata->num_mboxes = num_mboxes; mbox = pdata->ipi_mboxes; + mbox->setup_ipi_fn = ipi_fn; + for_each_available_child_of_node(np, nc) { mbox->pdata = pdata; ret = zynqmp_ipi_mbox_probe(mbox, nc); @@ -665,14 +948,32 @@ static int zynqmp_ipi_probe(struct platform_device *pdev) mbox++; } - /* IPI IRQ */ - ret = platform_get_irq(pdev, 0); - if (ret < 0) + ret = of_irq_parse_one(dev_of_node(dev), 0, &out_irq); + if (ret < 0) { + dev_err(dev, "failed to parse interrupts\n"); goto free_mbox_dev; + } + ret = out_irq.args[1]; + + /* + * If Interrupt number is in SGI range, then request SGI else request + * IPI system IRQ. + */ + if (ret < MAX_SGI) { + pdata->irq = ret; + ret = xlnx_mbox_init_sgi(pdev, pdata->irq, pdata); + if (ret) + goto free_mbox_dev; + } else { + ret = platform_get_irq(pdev, 0); + if (ret < 0) + goto free_mbox_dev; + + pdata->irq = ret; + ret = devm_request_irq(dev, pdata->irq, zynqmp_ipi_interrupt, + IRQF_SHARED, dev_name(dev), pdata); + } - pdata->irq = ret; - ret = devm_request_irq(dev, pdata->irq, zynqmp_ipi_interrupt, - IRQF_SHARED, dev_name(dev), pdata); if (ret) { dev_err(dev, "IRQ %d is not requested successfully.\n", pdata->irq); @@ -695,6 +996,17 @@ static void zynqmp_ipi_remove(struct platform_device *pdev) zynqmp_ipi_free_mboxes(pdata); } +static const struct of_device_id zynqmp_ipi_of_match[] = { + { .compatible = "xlnx,zynqmp-ipi-mailbox", + .data = &zynqmp_ipi_setup, + }, + { .compatible = "xlnx,versal-ipi-mailbox", + .data = &versal_ipi_setup, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, zynqmp_ipi_of_match); + static struct platform_driver zynqmp_ipi_driver = { .probe = zynqmp_ipi_probe, .remove_new = zynqmp_ipi_remove, diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index cba09660148a..4d11fc664cb0 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -171,7 +171,7 @@ static const char *read_super(struct cache_sb *sb, struct block_device *bdev, struct page *page; unsigned int i; - page = read_cache_page_gfp(bdev->bd_inode->i_mapping, + page = read_cache_page_gfp(bdev->bd_mapping, SB_OFFSET >> PAGE_SHIFT, GFP_KERNEL); if (IS_ERR(page)) return "IO error"; @@ -2555,10 +2555,6 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr, if (IS_ERR(bdev_file)) goto out_free_sb; - err = "failed to set blocksize"; - if (set_blocksize(file_bdev(bdev_file), 4096)) - goto out_blkdev_put; - err = read_super(sb, file_bdev(bdev_file), &sb_disk); if (err) goto out_blkdev_put; diff --git a/drivers/md/dm-vdo/Makefile b/drivers/md/dm-vdo/Makefile index 33e09abc6acd..9476957bfbf4 100644 --- a/drivers/md/dm-vdo/Makefile +++ b/drivers/md/dm-vdo/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I$(srctree)/$(src) -I$(srctree)/$(src)/indexer +ccflags-y := -I$(src) -I$(src)/indexer obj-$(CONFIG_DM_VDO) += dm-vdo.o diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 5a4b0a927f56..b423bec6458b 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -878,7 +878,7 @@ static int parse_device_config(int argc, char **argv, struct dm_target *ti, } if (config->version == 0) { - u64 device_size = i_size_read(config->owned_device->bdev->bd_inode); + u64 device_size = bdev_nr_bytes(config->owned_device->bdev); config->physical_blocks = device_size / VDO_BLOCK_SIZE; } @@ -1011,7 +1011,7 @@ static void vdo_status(struct dm_target *ti, status_type_t status_type, static block_count_t __must_check get_underlying_device_block_count(const struct vdo *vdo) { - return i_size_read(vdo_get_backing_device(vdo)->bd_inode) / VDO_BLOCK_SIZE; + return bdev_nr_bytes(vdo_get_backing_device(vdo)) / VDO_BLOCK_SIZE; } static int __must_check process_vdo_message_locked(struct vdo *vdo, unsigned int argc, diff --git a/drivers/md/dm-vdo/indexer/io-factory.c b/drivers/md/dm-vdo/indexer/io-factory.c index 515765d35794..1bee9d63dc0a 100644 --- a/drivers/md/dm-vdo/indexer/io-factory.c +++ b/drivers/md/dm-vdo/indexer/io-factory.c @@ -90,7 +90,7 @@ void uds_put_io_factory(struct io_factory *factory) size_t uds_get_writable_size(struct io_factory *factory) { - return i_size_read(factory->bdev->bd_inode); + return bdev_nr_bytes(factory->bdev); } /* Create a struct dm_bufio_client for an index region starting at offset. */ diff --git a/drivers/media/cec/platform/sti/stih-cec.c b/drivers/media/cec/platform/sti/stih-cec.c index a20fc5c0c88d..99978a7c8d9b 100644 --- a/drivers/media/cec/platform/sti/stih-cec.c +++ b/drivers/media/cec/platform/sti/stih-cec.c @@ -6,6 +6,7 @@ */ #include <linux/clk.h> #include <linux/interrupt.h> +#include <linux/io.h> #include <linux/kernel.h> #include <linux/mfd/syscon.h> #include <linux/module.h> diff --git a/drivers/media/dvb-frontends/af9013.c b/drivers/media/dvb-frontends/af9013.c index a829c89792a4..5afdbe244596 100644 --- a/drivers/media/dvb-frontends/af9013.c +++ b/drivers/media/dvb-frontends/af9013.c @@ -1480,7 +1480,7 @@ static int af9013_probe(struct i2c_client *client) goto err_regmap_exit; } state->muxc->priv = state; - ret = i2c_mux_add_adapter(state->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(state->muxc, 0, 0); if (ret) goto err_regmap_exit; diff --git a/drivers/media/dvb-frontends/lgdt3306a.c b/drivers/media/dvb-frontends/lgdt3306a.c index 231b45632ad5..b25d11be8611 100644 --- a/drivers/media/dvb-frontends/lgdt3306a.c +++ b/drivers/media/dvb-frontends/lgdt3306a.c @@ -2208,7 +2208,7 @@ static int lgdt3306a_probe(struct i2c_client *client) goto err_kfree; } state->muxc->priv = client; - ret = i2c_mux_add_adapter(state->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(state->muxc, 0, 0); if (ret) goto err_kfree; diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c index 389548fa2e0c..5a03485686d9 100644 --- a/drivers/media/dvb-frontends/m88ds3103.c +++ b/drivers/media/dvb-frontends/m88ds3103.c @@ -1873,7 +1873,7 @@ static int m88ds3103_probe(struct i2c_client *client) goto err_kfree; } dev->muxc->priv = dev; - ret = i2c_mux_add_adapter(dev->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(dev->muxc, 0, 0); if (ret) goto err_kfree; diff --git a/drivers/media/dvb-frontends/rtl2830.c b/drivers/media/dvb-frontends/rtl2830.c index 35c969fd2cb5..30d10fe4b33e 100644 --- a/drivers/media/dvb-frontends/rtl2830.c +++ b/drivers/media/dvb-frontends/rtl2830.c @@ -838,7 +838,7 @@ static int rtl2830_probe(struct i2c_client *client) goto err_regmap_exit; } dev->muxc->priv = client; - ret = i2c_mux_add_adapter(dev->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(dev->muxc, 0, 0); if (ret) goto err_regmap_exit; diff --git a/drivers/media/dvb-frontends/rtl2832.c b/drivers/media/dvb-frontends/rtl2832.c index 601cf45c3935..5142820b1b3d 100644 --- a/drivers/media/dvb-frontends/rtl2832.c +++ b/drivers/media/dvb-frontends/rtl2832.c @@ -1082,7 +1082,7 @@ static int rtl2832_probe(struct i2c_client *client) goto err_regmap_exit; } dev->muxc->priv = dev; - ret = i2c_mux_add_adapter(dev->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(dev->muxc, 0, 0); if (ret) goto err_regmap_exit; diff --git a/drivers/media/dvb-frontends/si2168.c b/drivers/media/dvb-frontends/si2168.c index dae1f2153e8b..26828fd41e68 100644 --- a/drivers/media/dvb-frontends/si2168.c +++ b/drivers/media/dvb-frontends/si2168.c @@ -744,7 +744,7 @@ static int si2168_probe(struct i2c_client *client) goto err_kfree; } dev->muxc->priv = client; - ret = i2c_mux_add_adapter(dev->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(dev->muxc, 0, 0); if (ret) goto err_kfree; diff --git a/drivers/media/i2c/max9286.c b/drivers/media/i2c/max9286.c index d685d445cf23..dfcb3fc03350 100644 --- a/drivers/media/i2c/max9286.c +++ b/drivers/media/i2c/max9286.c @@ -383,7 +383,7 @@ static int max9286_i2c_mux_init(struct max9286_priv *priv) for_each_source(priv, source) { unsigned int index = to_index(priv, source); - ret = i2c_mux_add_adapter(priv->mux, 0, index, 0); + ret = i2c_mux_add_adapter(priv->mux, 0, index); if (ret < 0) goto error; } diff --git a/drivers/media/rc/mtk-cir.c b/drivers/media/rc/mtk-cir.c index 4e294e59d3cb..b2f82b2d1c8d 100644 --- a/drivers/media/rc/mtk-cir.c +++ b/drivers/media/rc/mtk-cir.c @@ -8,6 +8,7 @@ #include <linux/clk.h> #include <linux/interrupt.h> #include <linux/module.h> +#include <linux/io.h> #include <linux/of.h> #include <linux/platform_device.h> #include <linux/reset.h> diff --git a/drivers/media/rc/serial_ir.c b/drivers/media/rc/serial_ir.c index 96ae0294ac10..fc5fd3927177 100644 --- a/drivers/media/rc/serial_ir.c +++ b/drivers/media/rc/serial_ir.c @@ -18,6 +18,7 @@ #include <linux/module.h> #include <linux/errno.h> #include <linux/interrupt.h> +#include <linux/io.h> #include <linux/kernel.h> #include <linux/serial_reg.h> #include <linux/types.h> diff --git a/drivers/media/rc/st_rc.c b/drivers/media/rc/st_rc.c index 28477aa95563..988b09191c4c 100644 --- a/drivers/media/rc/st_rc.c +++ b/drivers/media/rc/st_rc.c @@ -6,6 +6,7 @@ #include <linux/kernel.h> #include <linux/clk.h> #include <linux/interrupt.h> +#include <linux/io.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> diff --git a/drivers/media/rc/sunxi-cir.c b/drivers/media/rc/sunxi-cir.c index bf58c965ead8..b49df8355e6b 100644 --- a/drivers/media/rc/sunxi-cir.c +++ b/drivers/media/rc/sunxi-cir.c @@ -12,6 +12,7 @@ #include <linux/clk.h> #include <linux/interrupt.h> +#include <linux/io.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> diff --git a/drivers/media/usb/cx231xx/cx231xx-i2c.c b/drivers/media/usb/cx231xx/cx231xx-i2c.c index c6659253c6fb..6da8e7943d94 100644 --- a/drivers/media/usb/cx231xx/cx231xx-i2c.c +++ b/drivers/media/usb/cx231xx/cx231xx-i2c.c @@ -567,10 +567,7 @@ int cx231xx_i2c_mux_create(struct cx231xx *dev) int cx231xx_i2c_mux_register(struct cx231xx *dev, int mux_no) { - return i2c_mux_add_adapter(dev->muxc, - 0, - mux_no /* chan_id */, - 0 /* class */); + return i2c_mux_add_adapter(dev->muxc, 0, mux_no); } void cx231xx_i2c_mux_unregister(struct cx231xx *dev) diff --git a/drivers/memory/pl353-smc.c b/drivers/memory/pl353-smc.c index 48540817e046..56e51737c81f 100644 --- a/drivers/memory/pl353-smc.c +++ b/drivers/memory/pl353-smc.c @@ -154,7 +154,6 @@ MODULE_DEVICE_TABLE(amba, pl353_ids); static struct amba_driver pl353_smc_driver = { .drv = { - .owner = THIS_MODULE, .name = "pl353-smc", .pm = &pl353_smc_dev_pm_ops, }, diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c index 8c00e0c695c5..9f4782bdbf4b 100644 --- a/drivers/mfd/intel-lpss-pci.c +++ b/drivers/mfd/intel-lpss-pci.c @@ -54,7 +54,7 @@ static int intel_lpss_pci_probe(struct pci_dev *pdev, if (ret) return ret; - ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_LEGACY); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_INTX); if (ret < 0) return ret; diff --git a/drivers/misc/lkdtm/Makefile b/drivers/misc/lkdtm/Makefile index b28701138b4b..39468bd27b85 100644 --- a/drivers/misc/lkdtm/Makefile +++ b/drivers/misc/lkdtm/Makefile @@ -15,10 +15,6 @@ lkdtm-$(CONFIG_PPC_64S_HASH_MMU) += powerpc.o KASAN_SANITIZE_stackleak.o := n -KASAN_SANITIZE_rodata.o := n -KCSAN_SANITIZE_rodata.o := n -KCOV_INSTRUMENT_rodata.o := n -OBJECT_FILES_NON_STANDARD_rodata.o := y CFLAGS_REMOVE_rodata.o += $(CC_FLAGS_LTO) $(RETHUNK_CFLAGS) $(CC_FLAGS_CFI) OBJCOPYFLAGS := diff --git a/drivers/misc/vmw_vmci/vmci_guest.c b/drivers/misc/vmw_vmci/vmci_guest.c index 4f8d962bb5b2..c61e8953511d 100644 --- a/drivers/misc/vmw_vmci/vmci_guest.c +++ b/drivers/misc/vmw_vmci/vmci_guest.c @@ -787,8 +787,7 @@ static int vmci_guest_probe_device(struct pci_dev *pdev, error = pci_alloc_irq_vectors(pdev, num_irq_vectors, num_irq_vectors, PCI_IRQ_MSIX); if (error < 0) { - error = pci_alloc_irq_vectors(pdev, 1, 1, - PCI_IRQ_MSIX | PCI_IRQ_MSI | PCI_IRQ_LEGACY); + error = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); if (error < 0) goto err_unsubscribe_event; } else { diff --git a/drivers/mtd/devices/block2mtd.c b/drivers/mtd/devices/block2mtd.c index caacdc0a3819..b06c8dd51562 100644 --- a/drivers/mtd/devices/block2mtd.c +++ b/drivers/mtd/devices/block2mtd.c @@ -265,6 +265,7 @@ static struct block2mtd_dev *add_device(char *devname, int erase_size, struct file *bdev_file; struct block_device *bdev; struct block2mtd_dev *dev; + loff_t size; char *name; if (!devname) @@ -291,7 +292,8 @@ static struct block2mtd_dev *add_device(char *devname, int erase_size, goto err_free_block2mtd; } - if ((long)bdev->bd_inode->i_size % erase_size) { + size = bdev_nr_bytes(bdev); + if ((long)size % erase_size) { pr_err("erasesize must be a divisor of device size\n"); goto err_free_block2mtd; } @@ -309,7 +311,7 @@ static struct block2mtd_dev *add_device(char *devname, int erase_size, dev->mtd.name = name; - dev->mtd.size = bdev->bd_inode->i_size & PAGE_MASK; + dev->mtd.size = size & PAGE_MASK; dev->mtd.erasesize = erase_size; dev->mtd.writesize = 1; dev->mtd.writebufsize = PAGE_SIZE; diff --git a/drivers/mux/core.c b/drivers/mux/core.c index 775816112932..78c0022697ec 100644 --- a/drivers/mux/core.c +++ b/drivers/mux/core.c @@ -64,7 +64,7 @@ static void mux_chip_release(struct device *dev) { struct mux_chip *mux_chip = to_mux_chip(dev); - ida_simple_remove(&mux_ida, mux_chip->id); + ida_free(&mux_ida, mux_chip->id); kfree(mux_chip); } @@ -111,7 +111,7 @@ struct mux_chip *mux_chip_alloc(struct device *dev, mux_chip->dev.of_node = dev->of_node; dev_set_drvdata(&mux_chip->dev, mux_chip); - mux_chip->id = ida_simple_get(&mux_ida, 0, 0, GFP_KERNEL); + mux_chip->id = ida_alloc(&mux_ida, GFP_KERNEL); if (mux_chip->id < 0) { int err = mux_chip->id; diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 3b0c7758ea4f..3c3fcce4acd4 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -6477,16 +6477,16 @@ static int __init bonding_init(void) if (res) goto out; + bond_create_debugfs(); + res = register_pernet_subsys(&bond_net_ops); if (res) - goto out; + goto err_net_ops; res = bond_netlink_init(); if (res) goto err_link; - bond_create_debugfs(); - for (i = 0; i < max_bonds; i++) { res = bond_create(&init_net, NULL); if (res) @@ -6501,10 +6501,11 @@ static int __init bonding_init(void) out: return res; err: - bond_destroy_debugfs(); bond_netlink_fini(); err_link: unregister_pernet_subsys(&bond_net_ops); +err_net_ops: + bond_destroy_debugfs(); goto out; } @@ -6513,11 +6514,11 @@ static void __exit bonding_exit(void) { unregister_netdevice_notifier(&bond_netdev_notifier); - bond_destroy_debugfs(); - bond_netlink_fini(); unregister_pernet_subsys(&bond_net_ops); + bond_destroy_debugfs(); + #ifdef CONFIG_NET_POLL_CONTROLLER /* Make sure we don't have an imbalance on our netpoll blocking */ WARN_ON(atomic_read(&netpoll_block_tx)); diff --git a/drivers/net/dsa/microchip/ksz_dcb.c b/drivers/net/dsa/microchip/ksz_dcb.c index a97106327562..086bc9b3cf53 100644 --- a/drivers/net/dsa/microchip/ksz_dcb.c +++ b/drivers/net/dsa/microchip/ksz_dcb.c @@ -805,5 +805,15 @@ int ksz_dcb_init(struct ksz_device *dev) if (ret) return ret; + /* Enable 802.1p priority control on Port 2 during switch initialization. + * This setup is critical for the apptrust functionality on Port 1, which + * relies on the priority settings of Port 2. Note: Port 1 is naturally + * configured before Port 2, necessitating this configuration order. + */ + if (ksz_is_ksz88x3(dev)) + return ksz_prmw8(dev, KSZ_PORT_2, KSZ8_REG_PORT_1_CTRL_0, + KSZ8_PORT_802_1P_ENABLE, + KSZ8_PORT_802_1P_ENABLE); + return 0; } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c index f409d7bd1f1e..c5e5fac49779 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c @@ -170,7 +170,7 @@ static int xgbe_config_irqs(struct xgbe_prv_data *pdata) goto out; ret = pci_alloc_irq_vectors(pdata->pcidev, 1, 1, - PCI_IRQ_LEGACY | PCI_IRQ_MSI); + PCI_IRQ_INTX | PCI_IRQ_MSI); if (ret < 0) { dev_info(pdata->dev, "single IRQ enablement failed\n"); return ret; diff --git a/drivers/net/ethernet/aquantia/atlantic/Makefile b/drivers/net/ethernet/aquantia/atlantic/Makefile index 8ebcc68e807f..f6a96931c89a 100644 --- a/drivers/net/ethernet/aquantia/atlantic/Makefile +++ b/drivers/net/ethernet/aquantia/atlantic/Makefile @@ -8,7 +8,7 @@ obj-$(CONFIG_AQTION) += atlantic.o -ccflags-y += -I$(srctree)/$(src) +ccflags-y += -I$(src) atlantic-objs := aq_main.o \ aq_nic.o \ diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_cfg.h b/drivers/net/ethernet/aquantia/atlantic/aq_cfg.h index 7e9c74b141ef..fc2b325f34e7 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_cfg.h +++ b/drivers/net/ethernet/aquantia/atlantic/aq_cfg.h @@ -17,7 +17,7 @@ #define AQ_CFG_IS_POLLING_DEF 0U -#define AQ_CFG_FORCE_LEGACY_INT 0U +#define AQ_CFG_FORCE_INTX 0U #define AQ_CFG_INTERRUPT_MODERATION_OFF 0 #define AQ_CFG_INTERRUPT_MODERATION_ON 1 diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h index dbd284660135..f010bda61c96 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h +++ b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h @@ -104,7 +104,7 @@ struct aq_stats_s { }; #define AQ_HW_IRQ_INVALID 0U -#define AQ_HW_IRQ_LEGACY 1U +#define AQ_HW_IRQ_INTX 1U #define AQ_HW_IRQ_MSI 2U #define AQ_HW_IRQ_MSIX 3U diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c index d6d6d5d37ff3..fe0e3e2a8117 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c @@ -127,7 +127,7 @@ void aq_nic_cfg_start(struct aq_nic_s *self) cfg->irq_type = aq_pci_func_get_irq_type(self); - if ((cfg->irq_type == AQ_HW_IRQ_LEGACY) || + if ((cfg->irq_type == AQ_HW_IRQ_INTX) || (cfg->aq_hw_caps->vecs == 1U) || (cfg->vecs == 1U)) { cfg->is_rss = 0U; diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c index baa5f8cc31f2..43c71f6b314f 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c @@ -200,7 +200,7 @@ unsigned int aq_pci_func_get_irq_type(struct aq_nic_s *self) if (self->pdev->msi_enabled) return AQ_HW_IRQ_MSI; - return AQ_HW_IRQ_LEGACY; + return AQ_HW_IRQ_INTX; } static void aq_pci_free_irq_vectors(struct aq_nic_s *self) @@ -298,11 +298,8 @@ static int aq_pci_probe(struct pci_dev *pdev, numvecs += AQ_HW_SERVICE_IRQS; /*enable interrupts */ -#if !AQ_CFG_FORCE_LEGACY_INT - err = pci_alloc_irq_vectors(self->pdev, 1, numvecs, - PCI_IRQ_MSIX | PCI_IRQ_MSI | - PCI_IRQ_LEGACY); - +#if !AQ_CFG_FORCE_INTX + err = pci_alloc_irq_vectors(self->pdev, 1, numvecs, PCI_IRQ_ALL_TYPES); if (err < 0) goto err_hwinit; numvecs = err; diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c index 9dfd68f0fda9..8de2cdd09213 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c @@ -352,7 +352,7 @@ static int hw_atl_a0_hw_init(struct aq_hw_s *self, const u8 *mac_addr) { static u32 aq_hw_atl_igcr_table_[4][2] = { [AQ_HW_IRQ_INVALID] = { 0x20000000U, 0x20000000U }, - [AQ_HW_IRQ_LEGACY] = { 0x20000080U, 0x20000080U }, + [AQ_HW_IRQ_INTX] = { 0x20000080U, 0x20000080U }, [AQ_HW_IRQ_MSI] = { 0x20000021U, 0x20000025U }, [AQ_HW_IRQ_MSIX] = { 0x20000022U, 0x20000026U }, }; diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c index 54e70f07b573..56c46266bb0a 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c @@ -562,7 +562,7 @@ static int hw_atl_b0_hw_init(struct aq_hw_s *self, const u8 *mac_addr) { static u32 aq_hw_atl_igcr_table_[4][2] = { [AQ_HW_IRQ_INVALID] = { 0x20000000U, 0x20000000U }, - [AQ_HW_IRQ_LEGACY] = { 0x20000080U, 0x20000080U }, + [AQ_HW_IRQ_INTX] = { 0x20000080U, 0x20000080U }, [AQ_HW_IRQ_MSI] = { 0x20000021U, 0x20000025U }, [AQ_HW_IRQ_MSIX] = { 0x20000022U, 0x20000026U }, }; diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c index 220400a633f5..b0ed572e88c6 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c @@ -534,7 +534,7 @@ static int hw_atl2_hw_init(struct aq_hw_s *self, const u8 *mac_addr) { static u32 aq_hw_atl2_igcr_table_[4][2] = { [AQ_HW_IRQ_INVALID] = { 0x20000000U, 0x20000000U }, - [AQ_HW_IRQ_LEGACY] = { 0x20000080U, 0x20000080U }, + [AQ_HW_IRQ_INTX] = { 0x20000080U, 0x20000080U }, [AQ_HW_IRQ_MSI] = { 0x20000021U, 0x20000025U }, [AQ_HW_IRQ_MSIX] = { 0x20000022U, 0x20000026U }, }; diff --git a/drivers/net/ethernet/atheros/alx/main.c b/drivers/net/ethernet/atheros/alx/main.c index 3d28654e5df7..ad6d6abd885f 100644 --- a/drivers/net/ethernet/atheros/alx/main.c +++ b/drivers/net/ethernet/atheros/alx/main.c @@ -901,7 +901,7 @@ static int alx_init_intr(struct alx_priv *alx) int ret; ret = pci_alloc_irq_vectors(alx->hw.pdev, 1, 1, - PCI_IRQ_MSI | PCI_IRQ_LEGACY); + PCI_IRQ_MSI | PCI_IRQ_INTX); if (ret < 0) return ret; diff --git a/drivers/net/ethernet/chelsio/libcxgb/Makefile b/drivers/net/ethernet/chelsio/libcxgb/Makefile index aa79264e72ba..fbedc31674b3 100644 --- a/drivers/net/ethernet/chelsio/libcxgb/Makefile +++ b/drivers/net/ethernet/chelsio/libcxgb/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../cxgb4 +ccflags-y := -I $(src)/../cxgb4 obj-$(CONFIG_CHELSIO_LIB) += libcxgb.o diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c index 4b15af6b7122..44da335d66bd 100644 --- a/drivers/net/ethernet/engleder/tsnep_main.c +++ b/drivers/net/ethernet/engleder/tsnep_main.c @@ -1587,7 +1587,7 @@ static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi, length = __le32_to_cpu(entry->desc_wb->properties) & TSNEP_DESC_LENGTH_MASK; xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN); - xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool); + xsk_buff_dma_sync_for_cpu(entry->xdp); /* RX metadata with timestamps is in front of actual data, * subtract metadata size to get length of actual data and diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c index 051748b997f3..a466c2379146 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c @@ -55,7 +55,7 @@ static u32 dpaa2_xsk_run_xdp(struct dpaa2_eth_priv *priv, xdp_set_data_meta_invalid(xdp_buff); xdp_buff->rxq = &ch->xdp_rxq; - xsk_buff_dma_sync_for_cpu(xdp_buff, ch->xsk_pool); + xsk_buff_dma_sync_for_cpu(xdp_buff); xdp_act = bpf_prog_run_xdp(xdp_prog, xdp_buff); /* xdp.data pointer may have changed */ diff --git a/drivers/net/ethernet/fungible/funeth/Makefile b/drivers/net/ethernet/fungible/funeth/Makefile index 646d69595b4f..d51e4c2b4a1a 100644 --- a/drivers/net/ethernet/fungible/funeth/Makefile +++ b/drivers/net/ethernet/fungible/funeth/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) -ccflags-y += -I$(srctree)/$(src)/../funcore -I$(srctree)/$(src) +ccflags-y += -I$(src)/../funcore -I$(src) obj-$(CONFIG_FUN_ETH) += funeth.o diff --git a/drivers/net/ethernet/hisilicon/hns3/Makefile b/drivers/net/ethernet/hisilicon/hns3/Makefile index e214bfaece1f..8e9293e57bfd 100644 --- a/drivers/net/ethernet/hisilicon/hns3/Makefile +++ b/drivers/net/ethernet/hisilicon/hns3/Makefile @@ -3,7 +3,7 @@ # Makefile for the HISILICON network device drivers. # -ccflags-y += -I$(srctree)/$(src) +ccflags-y += -I$(src) ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3pf ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3vf ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3_common diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index dfdc0e032c07..ff71fb1eced9 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -2068,8 +2068,6 @@ static void hns3_tx_push_bd(struct hns3_enet_ring *ring, int num) __iowrite64_copy(ring->tqp->mem_base, desc, (sizeof(struct hns3_desc) * HNS3_MAX_PUSH_BD_NUM) / HNS3_BYTES_PER_64BIT); - - io_stop_wc(); } static void hns3_tx_mem_doorbell(struct hns3_enet_ring *ring) @@ -2088,8 +2086,6 @@ static void hns3_tx_mem_doorbell(struct hns3_enet_ring *ring) u64_stats_update_begin(&ring->syncp); ring->stats.tx_mem_doorbell += ring->pending_buf; u64_stats_update_end(&ring->syncp); - - io_stop_wc(); } static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num, diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index a85b425794df..4e885df789ef 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -482,7 +482,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) bi = *i40e_rx_bi(rx_ring, next_to_process); xsk_buff_set_size(bi, size); - xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool); + xsk_buff_dma_sync_for_cpu(bi); if (!first) first = bi; diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index aa81d1162b81..7541f223bf4f 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -878,7 +878,7 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) ICE_RX_FLX_DESC_PKT_LEN_M; xsk_buff_set_size(xdp, size); - xsk_buff_dma_sync_for_cpu(xdp, xsk_pool); + xsk_buff_dma_sync_for_cpu(xdp); if (!first) { first = xdp; diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index 986d429d1175..6972d728431c 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -376,7 +376,8 @@ static int idpf_set_ringparam(struct net_device *netdev, new_tx_count); if (new_tx_count == vport->txq_desc_count && - new_rx_count == vport->rxq_desc_count) + new_rx_count == vport->rxq_desc_count && + kring->tcp_data_split == idpf_vport_get_hsplit(vport)) goto unlock_mutex; if (!idpf_vport_set_hsplit(vport, kring->tcp_data_split)) { diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index b5bcabab7a1d..12f004f46082 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -2812,7 +2812,7 @@ static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget) } bi->xdp->data_end = bi->xdp->data + size; - xsk_buff_dma_sync_for_cpu(bi->xdp, ring->xsk_pool); + xsk_buff_dma_sync_for_cpu(bi->xdp); res = __igc_xdp_run_prog(adapter, prog, bi->xdp); switch (res) { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 397cb773fabb..3e3b471e53f0 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -303,7 +303,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, } bi->xdp->data_end = bi->xdp->data + size; - xsk_buff_dma_sync_for_cpu(bi->xdp, rx_ring->xsk_pool); + xsk_buff_dma_sync_for_cpu(bi->xdp); xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp); if (likely(xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR))) { diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c b/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c index 2e2c3be8a0b4..e6eb98d70f3c 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c @@ -15,6 +15,7 @@ #include <linux/io.h> #include <linux/pci.h> #include <linux/etherdevice.h> +#include <linux/vmalloc.h> #include "octep_config.h" #include "octep_main.h" diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_mbox.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_mbox.c index 2eab21e43048..445b626efe11 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_mbox.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_mbox.c @@ -7,6 +7,7 @@ #include <linux/types.h> #include <linux/pci.h> #include <linux/netdevice.h> +#include <linux/vmalloc.h> #include "octep_vf_config.h" #include "octep_vf_main.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c index b8dd74453655..1b7132fa70de 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c @@ -270,7 +270,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, /* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */ mxbuf->cqe = cqe; xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt); - xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool); + xsk_buff_dma_sync_for_cpu(&mxbuf->xdp); net_prefetch(mxbuf->xdp.data); /* Possible flows: @@ -319,7 +319,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, /* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */ mxbuf->cqe = cqe; xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt); - xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool); + xsk_buff_dma_sync_for_cpu(&mxbuf->xdp); net_prefetch(mxbuf->xdp.data); prog = rcu_dereference(rq->xdp_prog); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index d601b5faaed5..b5333da20e8a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -917,7 +917,7 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) if (!rq->xsk_pool) { count = mlx5e_refill_rx_wqes(rq, head, wqe_bulk); - } else if (likely(!rq->xsk_pool->dma_need_sync)) { + } else if (likely(!dma_dev_need_sync(rq->pdev))) { mlx5e_xsk_free_rx_wqes(rq, head, wqe_bulk); count = mlx5e_xsk_alloc_rx_wqes_batched(rq, head, wqe_bulk); } else { diff --git a/drivers/net/ethernet/micrel/ks8851_common.c b/drivers/net/ethernet/micrel/ks8851_common.c index 502518cdb461..6453c92f0fa7 100644 --- a/drivers/net/ethernet/micrel/ks8851_common.c +++ b/drivers/net/ethernet/micrel/ks8851_common.c @@ -328,7 +328,6 @@ static irqreturn_t ks8851_irq(int irq, void *_ks) { struct ks8851_net *ks = _ks; struct sk_buff_head rxq; - unsigned handled = 0; unsigned long flags; unsigned int status; struct sk_buff *skb; @@ -336,24 +335,17 @@ static irqreturn_t ks8851_irq(int irq, void *_ks) ks8851_lock(ks, &flags); status = ks8851_rdreg16(ks, KS_ISR); + ks8851_wrreg16(ks, KS_ISR, status); netif_dbg(ks, intr, ks->netdev, "%s: status 0x%04x\n", __func__, status); - if (status & IRQ_LCI) - handled |= IRQ_LCI; - if (status & IRQ_LDI) { u16 pmecr = ks8851_rdreg16(ks, KS_PMECR); pmecr &= ~PMECR_WKEVT_MASK; ks8851_wrreg16(ks, KS_PMECR, pmecr | PMECR_WKEVT_LINK); - - handled |= IRQ_LDI; } - if (status & IRQ_RXPSI) - handled |= IRQ_RXPSI; - if (status & IRQ_TXI) { unsigned short tx_space = ks8851_rdreg16(ks, KS_TXMIR); @@ -365,20 +357,12 @@ static irqreturn_t ks8851_irq(int irq, void *_ks) if (netif_queue_stopped(ks->netdev)) netif_wake_queue(ks->netdev); spin_unlock(&ks->statelock); - - handled |= IRQ_TXI; } - if (status & IRQ_RXI) - handled |= IRQ_RXI; - if (status & IRQ_SPIBEI) { netdev_err(ks->netdev, "%s: spi bus error\n", __func__); - handled |= IRQ_SPIBEI; } - ks8851_wrreg16(ks, KS_ISR, handled); - if (status & IRQ_RXI) { /* the datasheet says to disable the rx interrupt during * packet read-out, however we're masking the interrupt diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index b12d3b8a64fd..a3b919a3ce07 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -1087,8 +1087,6 @@ static int lan966x_probe(struct platform_device *pdev) platform_set_drvdata(pdev, lan966x); lan966x->dev = &pdev->dev; - lan966x->debugfs_root = debugfs_create_dir("lan966x", NULL); - if (!device_get_mac_address(&pdev->dev, mac_addr)) { ether_addr_copy(lan966x->base_mac, mac_addr); } else { @@ -1179,6 +1177,8 @@ static int lan966x_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, -ENODEV, "no ethernet-ports child found\n"); + lan966x->debugfs_root = debugfs_create_dir("lan966x", NULL); + /* init switch */ lan966x_init(lan966x); lan966x_stats_init(lan966x); @@ -1257,6 +1257,8 @@ cleanup_ports: destroy_workqueue(lan966x->stats_queue); mutex_destroy(&lan966x->stats_lock); + debugfs_remove_recursive(lan966x->debugfs_root); + return err; } diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c index 2729a2c5acf9..11021c34e47e 100644 --- a/drivers/net/ethernet/microsoft/mana/hw_channel.c +++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c @@ -3,6 +3,7 @@ #include <net/mana/gdma.h> #include <net/mana/hw_channel.h> +#include <linux/vmalloc.h> static int mana_hwc_get_msg_index(struct hw_channel_context *hwc, u16 *msg_id) { diff --git a/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c b/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c index 45be6954d5aa..01cfa9cc1b5e 100644 --- a/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c +++ b/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c @@ -184,7 +184,7 @@ nfp_nfd3_xsk_rx(struct nfp_net_rx_ring *rx_ring, int budget, xrxbuf->xdp->data += meta_len; xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len; xdp_set_data_meta_invalid(xrxbuf->xdp); - xsk_buff_dma_sync_for_cpu(xrxbuf->xdp, r_vec->xsk_pool); + xsk_buff_dma_sync_for_cpu(xrxbuf->xdp); net_prefetch(xrxbuf->xdp->data); if (meta_len) { diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 5abbea91bc07..e5ea827a21e7 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -5106,7 +5106,7 @@ static int rtl_alloc_irq(struct rtl8169_private *tp) rtl_lock_config_regs(tp); fallthrough; case RTL_GIGA_MAC_VER_07 ... RTL_GIGA_MAC_VER_17: - flags = PCI_IRQ_LEGACY; + flags = PCI_IRQ_INTX; break; default: flags = PCI_IRQ_ALL_TYPES; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 2e9a2da605f6..b3afc7cb7d72 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -5361,7 +5361,7 @@ read_again: /* RX buffer is good and fit into a XSK pool buffer */ buf->xdp->data_end = buf->xdp->data + buf1_len; - xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); + xsk_buff_dma_sync_for_cpu(buf->xdp); prog = READ_ONCE(priv->xdp_prog); res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c index 3662483bfe2e..7c4b6881a93f 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c @@ -1958,6 +1958,8 @@ int wx_sw_init(struct wx *wx) return -ENOMEM; } + bitmap_zero(wx->state, WX_STATE_NBITS); + return 0; } EXPORT_SYMBOL(wx_sw_init); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c index 6fae161cbcb8..68bde91b67a0 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c @@ -1674,14 +1674,14 @@ static int wx_set_interrupt_capability(struct wx *wx) /* minmum one for queue, one for misc*/ nvecs = 1; nvecs = pci_alloc_irq_vectors(pdev, nvecs, - nvecs, PCI_IRQ_MSI | PCI_IRQ_LEGACY); + nvecs, PCI_IRQ_MSI | PCI_IRQ_INTX); if (nvecs == 1) { if (pdev->msi_enabled) wx_err(wx, "Fallback to MSI.\n"); else - wx_err(wx, "Fallback to LEGACY.\n"); + wx_err(wx, "Fallback to INTx.\n"); } else { - wx_err(wx, "Failed to allocate MSI/LEGACY interrupts. Error: %d\n", nvecs); + wx_err(wx, "Failed to allocate MSI/INTx interrupts. Error: %d\n", nvecs); return nvecs; } @@ -2127,7 +2127,7 @@ void wx_write_eitr(struct wx_q_vector *q_vector) * wx_configure_vectors - Configure vectors for hardware * @wx: board private structure * - * wx_configure_vectors sets up the hardware to properly generate MSI-X/MSI/LEGACY + * wx_configure_vectors sets up the hardware to properly generate MSI-X/MSI/INTx * interrupts. **/ void wx_configure_vectors(struct wx *wx) @@ -2690,15 +2690,63 @@ int wx_set_features(struct net_device *netdev, netdev_features_t features) wx->rss_enabled = false; } - if (changed & - (NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_STAG_RX)) + netdev->features = features; + + if (wx->mac.type == wx_mac_sp && changed & NETIF_F_HW_VLAN_CTAG_RX) + wx->do_reset(netdev); + else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER)) wx_set_rx_mode(netdev); - return 1; + return 0; } EXPORT_SYMBOL(wx_set_features); +#define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ + NETIF_F_HW_VLAN_STAG_RX) + +#define NETIF_VLAN_INSERTION_FEATURES (NETIF_F_HW_VLAN_CTAG_TX | \ + NETIF_F_HW_VLAN_STAG_TX) + +#define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \ + NETIF_F_HW_VLAN_STAG_FILTER) + +netdev_features_t wx_fix_features(struct net_device *netdev, + netdev_features_t features) +{ + netdev_features_t changed = netdev->features ^ features; + struct wx *wx = netdev_priv(netdev); + + if (changed & NETIF_VLAN_STRIPPING_FEATURES) { + if ((features & NETIF_VLAN_STRIPPING_FEATURES) != NETIF_VLAN_STRIPPING_FEATURES && + (features & NETIF_VLAN_STRIPPING_FEATURES) != 0) { + features &= ~NETIF_VLAN_STRIPPING_FEATURES; + features |= netdev->features & NETIF_VLAN_STRIPPING_FEATURES; + wx_err(wx, "802.1Q and 802.1ad VLAN stripping must be either both on or both off."); + } + } + + if (changed & NETIF_VLAN_INSERTION_FEATURES) { + if ((features & NETIF_VLAN_INSERTION_FEATURES) != NETIF_VLAN_INSERTION_FEATURES && + (features & NETIF_VLAN_INSERTION_FEATURES) != 0) { + features &= ~NETIF_VLAN_INSERTION_FEATURES; + features |= netdev->features & NETIF_VLAN_INSERTION_FEATURES; + wx_err(wx, "802.1Q and 802.1ad VLAN insertion must be either both on or both off."); + } + } + + if (changed & NETIF_VLAN_FILTERING_FEATURES) { + if ((features & NETIF_VLAN_FILTERING_FEATURES) != NETIF_VLAN_FILTERING_FEATURES && + (features & NETIF_VLAN_FILTERING_FEATURES) != 0) { + features &= ~NETIF_VLAN_FILTERING_FEATURES; + features |= netdev->features & NETIF_VLAN_FILTERING_FEATURES; + wx_err(wx, "802.1Q and 802.1ad VLAN filtering must be either both on or both off."); + } + } + + return features; +} +EXPORT_SYMBOL(wx_fix_features); + void wx_set_ring(struct wx *wx, u32 new_tx_count, u32 new_rx_count, struct wx_ring *temp_ring) { diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.h b/drivers/net/ethernet/wangxun/libwx/wx_lib.h index ec909e876720..c41b29ea812f 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.h @@ -30,6 +30,8 @@ int wx_setup_resources(struct wx *wx); void wx_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats); int wx_set_features(struct net_device *netdev, netdev_features_t features); +netdev_features_t wx_fix_features(struct net_device *netdev, + netdev_features_t features); void wx_set_ring(struct wx *wx, u32 new_tx_count, u32 new_rx_count, struct wx_ring *temp_ring); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h index 1fdeb464d5f4..5aaf7b1fa2db 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_type.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h @@ -982,8 +982,13 @@ struct wx_hw_stats { u64 qmprc; }; +enum wx_state { + WX_STATE_RESETTING, + WX_STATE_NBITS, /* must be last */ +}; struct wx { unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; + DECLARE_BITMAP(state, WX_STATE_NBITS); void *priv; u8 __iomem *hw_addr; @@ -1071,6 +1076,8 @@ struct wx { u64 hw_csum_rx_good; u64 hw_csum_rx_error; u64 alloc_rx_buff_failed; + + void (*do_reset)(struct net_device *netdev); }; #define WX_INTR_ALL (~0ULL) @@ -1131,4 +1138,19 @@ static inline struct wx *phylink_to_wx(struct phylink_config *config) return container_of(config, struct wx, phylink_config); } +static inline int wx_set_state_reset(struct wx *wx) +{ + u8 timeout = 50; + + while (test_and_set_bit(WX_STATE_RESETTING, wx->state)) { + timeout--; + if (!timeout) + return -EBUSY; + + usleep_range(1000, 2000); + } + + return 0; +} + #endif /* _WX_TYPE_H_ */ diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c index 786a652ae64f..46a5a3e95202 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c @@ -52,7 +52,7 @@ static int ngbe_set_ringparam(struct net_device *netdev, struct wx *wx = netdev_priv(netdev); u32 new_rx_count, new_tx_count; struct wx_ring *temp_ring; - int i; + int i, err = 0; new_tx_count = clamp_t(u32, ring->tx_pending, WX_MIN_TXD, WX_MAX_TXD); new_tx_count = ALIGN(new_tx_count, WX_REQ_TX_DESCRIPTOR_MULTIPLE); @@ -64,6 +64,10 @@ static int ngbe_set_ringparam(struct net_device *netdev, new_rx_count == wx->rx_ring_count) return 0; + err = wx_set_state_reset(wx); + if (err) + return err; + if (!netif_running(wx->netdev)) { for (i = 0; i < wx->num_tx_queues; i++) wx->tx_ring[i]->count = new_tx_count; @@ -72,14 +76,16 @@ static int ngbe_set_ringparam(struct net_device *netdev, wx->tx_ring_count = new_tx_count; wx->rx_ring_count = new_rx_count; - return 0; + goto clear_reset; } /* allocate temporary buffer to store rings in */ i = max_t(int, wx->num_tx_queues, wx->num_rx_queues); temp_ring = kvmalloc_array(i, sizeof(struct wx_ring), GFP_KERNEL); - if (!temp_ring) - return -ENOMEM; + if (!temp_ring) { + err = -ENOMEM; + goto clear_reset; + } ngbe_down(wx); @@ -89,7 +95,9 @@ static int ngbe_set_ringparam(struct net_device *netdev, wx_configure(wx); ngbe_up(wx); - return 0; +clear_reset: + clear_bit(WX_STATE_RESETTING, wx->state); + return err; } static int ngbe_set_channels(struct net_device *dev, diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c index fdd6b4f70b7a..e894e01d030d 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c @@ -499,6 +499,7 @@ static const struct net_device_ops ngbe_netdev_ops = { .ndo_start_xmit = wx_xmit_frame, .ndo_set_rx_mode = wx_set_rx_mode, .ndo_set_features = wx_set_features, + .ndo_fix_features = wx_fix_features, .ndo_validate_addr = eth_validate_addr, .ndo_set_mac_address = wx_set_mac, .ndo_get_stats64 = wx_get_stats64, diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c index db675512ce4d..31fde3fa7c6b 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c @@ -19,7 +19,7 @@ static int txgbe_set_ringparam(struct net_device *netdev, struct wx *wx = netdev_priv(netdev); u32 new_rx_count, new_tx_count; struct wx_ring *temp_ring; - int i; + int i, err = 0; new_tx_count = clamp_t(u32, ring->tx_pending, WX_MIN_TXD, WX_MAX_TXD); new_tx_count = ALIGN(new_tx_count, WX_REQ_TX_DESCRIPTOR_MULTIPLE); @@ -31,6 +31,10 @@ static int txgbe_set_ringparam(struct net_device *netdev, new_rx_count == wx->rx_ring_count) return 0; + err = wx_set_state_reset(wx); + if (err) + return err; + if (!netif_running(wx->netdev)) { for (i = 0; i < wx->num_tx_queues; i++) wx->tx_ring[i]->count = new_tx_count; @@ -39,14 +43,16 @@ static int txgbe_set_ringparam(struct net_device *netdev, wx->tx_ring_count = new_tx_count; wx->rx_ring_count = new_rx_count; - return 0; + goto clear_reset; } /* allocate temporary buffer to store rings in */ i = max_t(int, wx->num_tx_queues, wx->num_rx_queues); temp_ring = kvmalloc_array(i, sizeof(struct wx_ring), GFP_KERNEL); - if (!temp_ring) - return -ENOMEM; + if (!temp_ring) { + err = -ENOMEM; + goto clear_reset; + } txgbe_down(wx); @@ -55,7 +61,9 @@ static int txgbe_set_ringparam(struct net_device *netdev, txgbe_up(wx); - return 0; +clear_reset: + clear_bit(WX_STATE_RESETTING, wx->state); + return err; } static int txgbe_set_channels(struct net_device *dev, diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c index bd4624d14ca0..8c7a74981b90 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c @@ -269,6 +269,8 @@ static int txgbe_sw_init(struct wx *wx) wx->tx_work_limit = TXGBE_DEFAULT_TX_WORK; wx->rx_work_limit = TXGBE_DEFAULT_RX_WORK; + wx->do_reset = txgbe_do_reset; + return 0; } @@ -421,6 +423,34 @@ int txgbe_setup_tc(struct net_device *dev, u8 tc) return 0; } +static void txgbe_reinit_locked(struct wx *wx) +{ + int err = 0; + + netif_trans_update(wx->netdev); + + err = wx_set_state_reset(wx); + if (err) { + wx_err(wx, "wait device reset timeout\n"); + return; + } + + txgbe_down(wx); + txgbe_up(wx); + + clear_bit(WX_STATE_RESETTING, wx->state); +} + +void txgbe_do_reset(struct net_device *netdev) +{ + struct wx *wx = netdev_priv(netdev); + + if (netif_running(netdev)) + txgbe_reinit_locked(wx); + else + txgbe_reset(wx); +} + static const struct net_device_ops txgbe_netdev_ops = { .ndo_open = txgbe_open, .ndo_stop = txgbe_close, @@ -428,6 +458,7 @@ static const struct net_device_ops txgbe_netdev_ops = { .ndo_start_xmit = wx_xmit_frame, .ndo_set_rx_mode = wx_set_rx_mode, .ndo_set_features = wx_set_features, + .ndo_fix_features = wx_fix_features, .ndo_validate_addr = eth_validate_addr, .ndo_set_mac_address = wx_set_mac, .ndo_get_stats64 = wx_get_stats64, diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h index 1b4ff50d5857..f434a7865cb7 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h @@ -134,6 +134,7 @@ extern char txgbe_driver_name[]; void txgbe_down(struct wx *wx); void txgbe_up(struct wx *wx); int txgbe_setup_tc(struct net_device *dev, u8 tc); +void txgbe_do_reset(struct net_device *netdev); #define NODE_PROP(_NAME, _PROP) \ (const struct software_node) { \ diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 19a9b50646c7..4e1a0fc0d555 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2902,14 +2902,14 @@ static void virtnet_rx_mode_work(struct work_struct *work) if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) return; - rtnl_lock(); - - promisc_allmulti = kzalloc(sizeof(*promisc_allmulti), GFP_ATOMIC); + promisc_allmulti = kzalloc(sizeof(*promisc_allmulti), GFP_KERNEL); if (!promisc_allmulti) { dev_warn(&dev->dev, "Failed to set RX mode, no memory.\n"); return; } + rtnl_lock(); + *promisc_allmulti = !!(dev->flags & IFF_PROMISC); sg_init_one(sg, promisc_allmulti, sizeof(*promisc_allmulti)); diff --git a/drivers/net/wireless/ath/ath10k/ahb.c b/drivers/net/wireless/ath/ath10k/ahb.c index a378bc48b1d2..f0441b3d7dcb 100644 --- a/drivers/net/wireless/ath/ath10k/ahb.c +++ b/drivers/net/wireless/ath/ath10k/ahb.c @@ -394,14 +394,14 @@ static irqreturn_t ath10k_ahb_interrupt_handler(int irq, void *arg) if (!ath10k_pci_irq_pending(ar)) return IRQ_NONE; - ath10k_pci_disable_and_clear_legacy_irq(ar); + ath10k_pci_disable_and_clear_intx_irq(ar); ath10k_pci_irq_msi_fw_mask(ar); napi_schedule(&ar->napi); return IRQ_HANDLED; } -static int ath10k_ahb_request_irq_legacy(struct ath10k *ar) +static int ath10k_ahb_request_irq_intx(struct ath10k *ar) { struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); struct ath10k_ahb *ar_ahb = ath10k_ahb_priv(ar); @@ -415,12 +415,12 @@ static int ath10k_ahb_request_irq_legacy(struct ath10k *ar) ar_ahb->irq, ret); return ret; } - ar_pci->oper_irq_mode = ATH10K_PCI_IRQ_LEGACY; + ar_pci->oper_irq_mode = ATH10K_PCI_IRQ_INTX; return 0; } -static void ath10k_ahb_release_irq_legacy(struct ath10k *ar) +static void ath10k_ahb_release_irq_intx(struct ath10k *ar) { struct ath10k_ahb *ar_ahb = ath10k_ahb_priv(ar); @@ -430,7 +430,7 @@ static void ath10k_ahb_release_irq_legacy(struct ath10k *ar) static void ath10k_ahb_irq_disable(struct ath10k *ar) { ath10k_ce_disable_interrupts(ar); - ath10k_pci_disable_and_clear_legacy_irq(ar); + ath10k_pci_disable_and_clear_intx_irq(ar); } static int ath10k_ahb_resource_init(struct ath10k *ar) @@ -621,7 +621,7 @@ static int ath10k_ahb_hif_start(struct ath10k *ar) ath10k_core_napi_enable(ar); ath10k_ce_enable_interrupts(ar); - ath10k_pci_enable_legacy_irq(ar); + ath10k_pci_enable_intx_irq(ar); ath10k_pci_rx_post(ar); @@ -775,7 +775,7 @@ static int ath10k_ahb_probe(struct platform_device *pdev) ath10k_pci_init_napi(ar); - ret = ath10k_ahb_request_irq_legacy(ar); + ret = ath10k_ahb_request_irq_intx(ar); if (ret) goto err_free_pipes; @@ -806,7 +806,7 @@ err_halt_device: ath10k_ahb_clock_disable(ar); err_free_irq: - ath10k_ahb_release_irq_legacy(ar); + ath10k_ahb_release_irq_intx(ar); err_free_pipes: ath10k_pci_release_resource(ar); @@ -828,7 +828,7 @@ static void ath10k_ahb_remove(struct platform_device *pdev) ath10k_core_unregister(ar); ath10k_ahb_irq_disable(ar); - ath10k_ahb_release_irq_legacy(ar); + ath10k_ahb_release_irq_intx(ar); ath10k_pci_release_resource(ar); ath10k_ahb_halt_chip(ar); ath10k_ahb_clock_disable(ar); diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c index 3777fe5d7d69..c52a16f8078f 100644 --- a/drivers/net/wireless/ath/ath10k/pci.c +++ b/drivers/net/wireless/ath/ath10k/pci.c @@ -721,7 +721,7 @@ bool ath10k_pci_irq_pending(struct ath10k *ar) return false; } -void ath10k_pci_disable_and_clear_legacy_irq(struct ath10k *ar) +void ath10k_pci_disable_and_clear_intx_irq(struct ath10k *ar) { /* IMPORTANT: INTR_CLR register has to be set after * INTR_ENABLE is set to 0, otherwise interrupt can not be @@ -739,7 +739,7 @@ void ath10k_pci_disable_and_clear_legacy_irq(struct ath10k *ar) PCIE_INTR_ENABLE_ADDRESS); } -void ath10k_pci_enable_legacy_irq(struct ath10k *ar) +void ath10k_pci_enable_intx_irq(struct ath10k *ar) { ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS + PCIE_INTR_ENABLE_ADDRESS, @@ -1935,7 +1935,7 @@ static void ath10k_pci_irq_msi_fw_unmask(struct ath10k *ar) static void ath10k_pci_irq_disable(struct ath10k *ar) { ath10k_ce_disable_interrupts(ar); - ath10k_pci_disable_and_clear_legacy_irq(ar); + ath10k_pci_disable_and_clear_intx_irq(ar); ath10k_pci_irq_msi_fw_mask(ar); } @@ -1949,7 +1949,7 @@ static void ath10k_pci_irq_sync(struct ath10k *ar) static void ath10k_pci_irq_enable(struct ath10k *ar) { ath10k_ce_enable_interrupts(ar); - ath10k_pci_enable_legacy_irq(ar); + ath10k_pci_enable_intx_irq(ar); ath10k_pci_irq_msi_fw_unmask(ar); } @@ -3111,11 +3111,11 @@ static irqreturn_t ath10k_pci_interrupt_handler(int irq, void *arg) return IRQ_NONE; } - if ((ar_pci->oper_irq_mode == ATH10K_PCI_IRQ_LEGACY) && + if ((ar_pci->oper_irq_mode == ATH10K_PCI_IRQ_INTX) && !ath10k_pci_irq_pending(ar)) return IRQ_NONE; - ath10k_pci_disable_and_clear_legacy_irq(ar); + ath10k_pci_disable_and_clear_intx_irq(ar); ath10k_pci_irq_msi_fw_mask(ar); napi_schedule(&ar->napi); @@ -3152,7 +3152,7 @@ static int ath10k_pci_napi_poll(struct napi_struct *ctx, int budget) napi_schedule(ctx); goto out; } - ath10k_pci_enable_legacy_irq(ar); + ath10k_pci_enable_intx_irq(ar); ath10k_pci_irq_msi_fw_unmask(ar); } @@ -3177,7 +3177,7 @@ static int ath10k_pci_request_irq_msi(struct ath10k *ar) return 0; } -static int ath10k_pci_request_irq_legacy(struct ath10k *ar) +static int ath10k_pci_request_irq_intx(struct ath10k *ar) { struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); int ret; @@ -3199,8 +3199,8 @@ static int ath10k_pci_request_irq(struct ath10k *ar) struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); switch (ar_pci->oper_irq_mode) { - case ATH10K_PCI_IRQ_LEGACY: - return ath10k_pci_request_irq_legacy(ar); + case ATH10K_PCI_IRQ_INTX: + return ath10k_pci_request_irq_intx(ar); case ATH10K_PCI_IRQ_MSI: return ath10k_pci_request_irq_msi(ar); default: @@ -3232,7 +3232,7 @@ static int ath10k_pci_init_irq(struct ath10k *ar) ath10k_pci_irq_mode); /* Try MSI */ - if (ath10k_pci_irq_mode != ATH10K_PCI_IRQ_LEGACY) { + if (ath10k_pci_irq_mode != ATH10K_PCI_IRQ_INTX) { ar_pci->oper_irq_mode = ATH10K_PCI_IRQ_MSI; ret = pci_enable_msi(ar_pci->pdev); if (ret == 0) @@ -3250,7 +3250,7 @@ static int ath10k_pci_init_irq(struct ath10k *ar) * For now, fix the race by repeating the write in below * synchronization checking. */ - ar_pci->oper_irq_mode = ATH10K_PCI_IRQ_LEGACY; + ar_pci->oper_irq_mode = ATH10K_PCI_IRQ_INTX; ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS + PCIE_INTR_ENABLE_ADDRESS, PCIE_INTR_FIRMWARE_MASK | PCIE_INTR_CE_MASK_ALL); @@ -3258,7 +3258,7 @@ static int ath10k_pci_init_irq(struct ath10k *ar) return 0; } -static void ath10k_pci_deinit_irq_legacy(struct ath10k *ar) +static void ath10k_pci_deinit_irq_intx(struct ath10k *ar) { ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS + PCIE_INTR_ENABLE_ADDRESS, 0); @@ -3269,8 +3269,8 @@ static int ath10k_pci_deinit_irq(struct ath10k *ar) struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); switch (ar_pci->oper_irq_mode) { - case ATH10K_PCI_IRQ_LEGACY: - ath10k_pci_deinit_irq_legacy(ar); + case ATH10K_PCI_IRQ_INTX: + ath10k_pci_deinit_irq_intx(ar); break; default: pci_disable_msi(ar_pci->pdev); @@ -3307,14 +3307,14 @@ int ath10k_pci_wait_for_target_init(struct ath10k *ar) if (val & FW_IND_INITIALIZED) break; - if (ar_pci->oper_irq_mode == ATH10K_PCI_IRQ_LEGACY) + if (ar_pci->oper_irq_mode == ATH10K_PCI_IRQ_INTX) /* Fix potential race by repeating CORE_BASE writes */ - ath10k_pci_enable_legacy_irq(ar); + ath10k_pci_enable_intx_irq(ar); mdelay(10); } while (time_before(jiffies, timeout)); - ath10k_pci_disable_and_clear_legacy_irq(ar); + ath10k_pci_disable_and_clear_intx_irq(ar); ath10k_pci_irq_msi_fw_mask(ar); if (val == 0xffffffff) { diff --git a/drivers/net/wireless/ath/ath10k/pci.h b/drivers/net/wireless/ath/ath10k/pci.h index 27bb4cf2dfea..4c3f536f2ea1 100644 --- a/drivers/net/wireless/ath/ath10k/pci.h +++ b/drivers/net/wireless/ath/ath10k/pci.h @@ -101,7 +101,7 @@ struct ath10k_pci_supp_chip { enum ath10k_pci_irq_mode { ATH10K_PCI_IRQ_AUTO = 0, - ATH10K_PCI_IRQ_LEGACY = 1, + ATH10K_PCI_IRQ_INTX = 1, ATH10K_PCI_IRQ_MSI = 2, }; @@ -243,9 +243,9 @@ int ath10k_pci_init_pipes(struct ath10k *ar); int ath10k_pci_init_config(struct ath10k *ar); void ath10k_pci_rx_post(struct ath10k *ar); void ath10k_pci_flush(struct ath10k *ar); -void ath10k_pci_enable_legacy_irq(struct ath10k *ar); +void ath10k_pci_enable_intx_irq(struct ath10k *ar); bool ath10k_pci_irq_pending(struct ath10k *ar); -void ath10k_pci_disable_and_clear_legacy_irq(struct ath10k *ar); +void ath10k_pci_disable_and_clear_intx_irq(struct ath10k *ar); void ath10k_pci_irq_msi_fw_mask(struct ath10k *ar); int ath10k_pci_wait_for_target_init(struct ath10k *ar); int ath10k_pci_setup_resource(struct ath10k *ar); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile index dc6d27a36faa..e5ca0f511822 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile @@ -6,8 +6,8 @@ # ccflags-y += \ - -I $(srctree)/$(src) \ - -I $(srctree)/$(src)/../include + -I $(src) \ + -I $(src)/../include obj-$(CONFIG_BRCMFMAC) += brcmfmac.o brcmfmac-objs += \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/Makefile index 46098705e236..5e37c638f966 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/Makefile @@ -3,9 +3,9 @@ # Copyright (c) 2022 Broadcom Corporation ccflags-y += \ - -I $(srctree)/$(src) \ - -I $(srctree)/$(src)/.. \ - -I $(srctree)/$(src)/../../include + -I $(src) \ + -I $(src)/.. \ + -I $(src)/../../include obj-m += brcmfmac-bca.o brcmfmac-bca-objs += \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/Makefile index 5e1fddaff79e..33e86724ba14 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/Makefile @@ -3,9 +3,9 @@ # Copyright (c) 2022 Broadcom Corporation ccflags-y += \ - -I $(srctree)/$(src) \ - -I $(srctree)/$(src)/.. \ - -I $(srctree)/$(src)/../../include + -I $(src) \ + -I $(src)/.. \ + -I $(src)/../../include obj-m += brcmfmac-cyw.o brcmfmac-cyw-objs += \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/Makefile index 7f455a19a2b1..3db4cce44f42 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/Makefile @@ -3,9 +3,9 @@ # Copyright (c) 2022 Broadcom Corporation ccflags-y += \ - -I $(srctree)/$(src) \ - -I $(srctree)/$(src)/.. \ - -I $(srctree)/$(src)/../../include + -I $(src) \ + -I $(src)/.. \ + -I $(src)/../../include obj-m += brcmfmac-wcc.o brcmfmac-wcc-objs += \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile index 090757730ba6..ffdd17eabe1d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile @@ -16,9 +16,9 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ccflags-y := \ - -I $(srctree)/$(src) \ - -I $(srctree)/$(src)/phy \ - -I $(srctree)/$(src)/../include + -I $(src) \ + -I $(src)/phy \ + -I $(src)/../include brcmsmac-y := \ mac80211_if.o \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmutil/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmutil/Makefile index 7a82d615ba2a..f9b40cab766c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmutil/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmutil/Makefile @@ -4,7 +4,7 @@ # # Copyright (c) 2011 Broadcom Corporation # -ccflags-y := -I $(srctree)/$(src)/../include +ccflags-y := -I $(src)/../include obj-$(CONFIG_BRCMUTIL) += brcmutil.o brcmutil-objs = utils.o d11.o diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/Makefile b/drivers/net/wireless/intel/iwlwifi/dvm/Makefile index 0486b17d7c41..6109d64006db 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/Makefile +++ b/drivers/net/wireless/intel/iwlwifi/dvm/Makefile @@ -11,4 +11,4 @@ iwldvm-objs += rxon.o devices.o iwldvm-$(CONFIG_IWLWIFI_LEDS) += led.o iwldvm-$(CONFIG_IWLWIFI_DEBUGFS) += debugfs.o -ccflags-y += -I $(srctree)/$(src)/../ +ccflags-y += -I $(src)/../ diff --git a/drivers/net/wireless/intel/iwlwifi/mei/Makefile b/drivers/net/wireless/intel/iwlwifi/mei/Makefile index 8e3ef0347db7..e05f9421be4a 100644 --- a/drivers/net/wireless/intel/iwlwifi/mei/Makefile +++ b/drivers/net/wireless/intel/iwlwifi/mei/Makefile @@ -5,4 +5,4 @@ iwlmei-y += net.o iwlmei-$(CONFIG_IWLWIFI_DEVICE_TRACING) += trace.o CFLAGS_trace.o := -I$(src) -ccflags-y += -I $(srctree)/$(src)/../ +ccflags-y += -I $(src)/../ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile index 5c754b87ea20..8873904f51ec 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile +++ b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile @@ -16,4 +16,4 @@ iwlmvm-$(CONFIG_IWLWIFI_LEDS) += led.o iwlmvm-$(CONFIG_PM) += d3.o iwlmvm-$(CONFIG_IWLMEI) += vendor-cmd.o -subdir-ccflags-y += -I $(srctree)/$(src)/../ +subdir-ccflags-y += -I $(src)/../ diff --git a/drivers/net/wireless/intel/iwlwifi/tests/Makefile b/drivers/net/wireless/intel/iwlwifi/tests/Makefile index 5658471bdf0a..84491488f589 100644 --- a/drivers/net/wireless/intel/iwlwifi/tests/Makefile +++ b/drivers/net/wireless/intel/iwlwifi/tests/Makefile @@ -2,6 +2,6 @@ iwlwifi-tests-y += module.o devinfo.o -ccflags-y += -I$(srctree)/$(src)/../ +ccflags-y += -I$(src)/../ obj-$(CONFIG_IWLWIFI_KUNIT_TESTS) += iwlwifi-tests.o diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8180/Makefile b/drivers/net/wireless/realtek/rtl818x/rtl8180/Makefile index 565a9a114134..1044105c2557 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8180/Makefile +++ b/drivers/net/wireless/realtek/rtl818x/rtl8180/Makefile @@ -3,4 +3,4 @@ rtl818x_pci-objs := dev.o rtl8225.o sa2400.o max2820.o grf5101.o rtl8225se.o obj-$(CONFIG_RTL8180) += rtl818x_pci.o -ccflags-y += -I $(srctree)/$(src)/.. +ccflags-y += -I $(src)/.. diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/Makefile b/drivers/net/wireless/realtek/rtl818x/rtl8187/Makefile index 0bf64dfb233a..3deded2c05b6 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8187/Makefile +++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/Makefile @@ -3,4 +3,4 @@ rtl8187-objs := dev.o rtl8225.o leds.o rfkill.o obj-$(CONFIG_RTL8187) += rtl8187.o -ccflags-y += -I $(srctree)/$(src)/.. +ccflags-y += -I $(src)/.. diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c index 7a093f3d5f74..30232f7e3ec5 100644 --- a/drivers/net/wireless/realtek/rtw88/pci.c +++ b/drivers/net/wireless/realtek/rtw88/pci.c @@ -1613,7 +1613,7 @@ static struct rtw_hci_ops rtw_pci_ops = { static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) { - unsigned int flags = PCI_IRQ_LEGACY; + unsigned int flags = PCI_IRQ_INTX; int ret; if (!rtw_disable_msi) diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index 7b00476a5dee..03bbcf9b6737 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -3637,7 +3637,7 @@ static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev, unsigned long flags = 0; int ret; - flags |= PCI_IRQ_LEGACY | PCI_IRQ_MSI; + flags |= PCI_IRQ_INTX | PCI_IRQ_MSI; ret = pci_alloc_irq_vectors(pdev, 1, 1, flags); if (ret < 0) { rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret); diff --git a/drivers/ntb/hw/idt/ntb_hw_idt.c b/drivers/ntb/hw/idt/ntb_hw_idt.c index 48823b53ede3..48dfb1a69a77 100644 --- a/drivers/ntb/hw/idt/ntb_hw_idt.c +++ b/drivers/ntb/hw/idt/ntb_hw_idt.c @@ -2129,7 +2129,7 @@ static int idt_init_isr(struct idt_ntb_dev *ndev) int ret; /* Allocate just one interrupt vector for the ISR */ - ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_LEGACY); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_INTX); if (ret != 1) { dev_err(&pdev->dev, "Failed to allocate IRQ vector"); return ret; diff --git a/drivers/of/address.c b/drivers/of/address.c index ae46a3605904..d669ce25b5f9 100644 --- a/drivers/of/address.c +++ b/drivers/of/address.c @@ -486,34 +486,30 @@ static int of_translate_one(struct device_node *parent, struct of_bus *bus, * device that had registered logical PIO mapping, and the return code is * relative to that node. */ -static u64 __of_translate_address(struct device_node *dev, +static u64 __of_translate_address(struct device_node *node, struct device_node *(*get_parent)(const struct device_node *), const __be32 *in_addr, const char *rprop, struct device_node **host) { - struct device_node *parent = NULL; + struct device_node *dev __free(device_node) = of_node_get(node); + struct device_node *parent __free(device_node) = get_parent(dev); struct of_bus *bus, *pbus; __be32 addr[OF_MAX_ADDR_CELLS]; int na, ns, pna, pns; - u64 result = OF_BAD_ADDR; pr_debug("** translation for device %pOF **\n", dev); - /* Increase refcount at current level */ - of_node_get(dev); - *host = NULL; - /* Get parent & match bus type */ - parent = get_parent(dev); + if (parent == NULL) - goto bail; + return OF_BAD_ADDR; bus = of_match_bus(parent); /* Count address cells & copy address locally */ bus->count_cells(dev, &na, &ns); if (!OF_CHECK_COUNTS(na, ns)) { pr_debug("Bad cell count for %pOF\n", dev); - goto bail; + return OF_BAD_ADDR; } memcpy(addr, in_addr, na * 4); @@ -533,8 +529,7 @@ static u64 __of_translate_address(struct device_node *dev, /* If root, we have finished */ if (parent == NULL) { pr_debug("reached root node\n"); - result = of_read_number(addr, na); - break; + return of_read_number(addr, na); } /* @@ -543,11 +538,11 @@ static u64 __of_translate_address(struct device_node *dev, */ iorange = find_io_range_by_fwnode(&dev->fwnode); if (iorange && (iorange->flags != LOGIC_PIO_CPU_MMIO)) { - result = of_read_number(addr + 1, na - 1); + u64 result = of_read_number(addr + 1, na - 1); pr_debug("indirectIO matched(%pOF) 0x%llx\n", dev, result); - *host = of_node_get(dev); - break; + *host = no_free_ptr(dev); + return result; } /* Get new parent bus and counts */ @@ -555,7 +550,7 @@ static u64 __of_translate_address(struct device_node *dev, pbus->count_cells(dev, &pna, &pns); if (!OF_CHECK_COUNTS(pna, pns)) { pr_err("Bad cell count for %pOF\n", dev); - break; + return OF_BAD_ADDR; } pr_debug("parent bus is %s (na=%d, ns=%d) on %pOF\n", @@ -563,7 +558,7 @@ static u64 __of_translate_address(struct device_node *dev, /* Apply bus translation */ if (of_translate_one(dev, bus, pbus, addr, na, ns, pna, rprop)) - break; + return OF_BAD_ADDR; /* Complete the move up one level */ na = pna; @@ -572,11 +567,8 @@ static u64 __of_translate_address(struct device_node *dev, of_dump_addr("one level translation:", addr, na); } - bail: - of_node_put(parent); - of_node_put(dev); - return result; + unreachable(); } u64 of_translate_address(struct device_node *dev, const __be32 *in_addr) @@ -654,19 +646,16 @@ EXPORT_SYMBOL(of_translate_dma_address); const __be32 *of_translate_dma_region(struct device_node *dev, const __be32 *prop, phys_addr_t *start, size_t *length) { - struct device_node *parent; + struct device_node *parent __free(device_node) = __of_get_dma_parent(dev); u64 address, size; int na, ns; - parent = __of_get_dma_parent(dev); if (!parent) return NULL; na = of_bus_n_addr_cells(parent); ns = of_bus_n_size_cells(parent); - of_node_put(parent); - address = of_translate_dma_address(dev, prop); if (address == OF_BAD_ADDR) return NULL; @@ -688,21 +677,19 @@ const __be32 *__of_get_address(struct device_node *dev, int index, int bar_no, { const __be32 *prop; unsigned int psize; - struct device_node *parent; + struct device_node *parent __free(device_node) = of_get_parent(dev); struct of_bus *bus; int onesize, i, na, ns; - /* Get parent & match bus type */ - parent = of_get_parent(dev); if (parent == NULL) return NULL; + + /* match the parent's bus type */ bus = of_match_bus(parent); - if (strcmp(bus->name, "pci") && (bar_no >= 0)) { - of_node_put(parent); + if (strcmp(bus->name, "pci") && (bar_no >= 0)) return NULL; - } + bus->count_cells(dev, &na, &ns); - of_node_put(parent); if (!OF_CHECK_ADDR_COUNT(na)) return NULL; @@ -888,14 +875,13 @@ static u64 of_translate_ioport(struct device_node *dev, const __be32 *in_addr, */ int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) { - struct device_node *node = of_node_get(np); + struct device_node *node __free(device_node) = of_node_get(np); const __be32 *ranges = NULL; bool found_dma_ranges = false; struct of_range_parser parser; struct of_range range; struct bus_dma_region *r; int len, num_ranges = 0; - int ret = 0; while (node) { ranges = of_get_property(node, "dma-ranges", &len); @@ -905,10 +891,9 @@ int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) break; /* Once we find 'dma-ranges', then a missing one is an error */ - if (found_dma_ranges && !ranges) { - ret = -ENODEV; - goto out; - } + if (found_dma_ranges && !ranges) + return -ENODEV; + found_dma_ranges = true; node = of_get_next_dma_parent(node); @@ -916,10 +901,8 @@ int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) if (!node || !ranges) { pr_debug("no dma-ranges found for node(%pOF)\n", np); - ret = -ENODEV; - goto out; + return -ENODEV; } - of_dma_range_parser_init(&parser, node); for_each_of_range(&parser, &range) { if (range.cpu_addr == OF_BAD_ADDR) { @@ -930,16 +913,12 @@ int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) num_ranges++; } - if (!num_ranges) { - ret = -EINVAL; - goto out; - } + if (!num_ranges) + return -EINVAL; r = kcalloc(num_ranges + 1, sizeof(*r), GFP_KERNEL); - if (!r) { - ret = -ENOMEM; - goto out; - } + if (!r) + return -ENOMEM; /* * Record all info in the generic DMA ranges array for struct device, @@ -957,9 +936,7 @@ int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) r->size = range.size; r++; } -out: - of_node_put(node); - return ret; + return 0; } #endif /* CONFIG_HAS_DMA */ @@ -1016,24 +993,18 @@ phys_addr_t __init of_dma_get_max_cpu_address(struct device_node *np) */ bool of_dma_is_coherent(struct device_node *np) { - struct device_node *node; - bool is_coherent = dma_default_coherent; - - node = of_node_get(np); + struct device_node *node __free(device_node) = of_node_get(np); while (node) { - if (of_property_read_bool(node, "dma-coherent")) { - is_coherent = true; - break; - } - if (of_property_read_bool(node, "dma-noncoherent")) { - is_coherent = false; - break; - } + if (of_property_read_bool(node, "dma-coherent")) + return true; + + if (of_property_read_bool(node, "dma-noncoherent")) + return false; + node = of_get_next_dma_parent(node); } - of_node_put(node); - return is_coherent; + return dma_default_coherent; } EXPORT_SYMBOL_GPL(of_dma_is_coherent); @@ -1049,20 +1020,14 @@ EXPORT_SYMBOL_GPL(of_dma_is_coherent); */ static bool of_mmio_is_nonposted(struct device_node *np) { - struct device_node *parent; - bool nonposted; - if (!IS_ENABLED(CONFIG_ARCH_APPLE)) return false; - parent = of_get_parent(np); + struct device_node *parent __free(device_node) = of_get_parent(np); if (!parent) return false; - nonposted = of_property_read_bool(parent, "nonposted-mmio"); - - of_node_put(parent); - return nonposted; + return of_property_read_bool(parent, "nonposted-mmio"); } static int __of_address_to_resource(struct device_node *dev, int index, int bar_no, diff --git a/drivers/of/base.c b/drivers/of/base.c index 8856c67c466a..20603d3c9931 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -16,6 +16,7 @@ #define pr_fmt(fmt) "OF: " fmt +#include <linux/cleanup.h> #include <linux/console.h> #include <linux/ctype.h> #include <linux/cpu.h> @@ -1393,8 +1394,10 @@ int of_parse_phandle_with_args_map(const struct device_node *np, const char *stem_name, int index, struct of_phandle_args *out_args) { - char *cells_name, *map_name = NULL, *mask_name = NULL; - char *pass_name = NULL; + char *cells_name __free(kfree) = kasprintf(GFP_KERNEL, "#%s-cells", stem_name); + char *map_name __free(kfree) = kasprintf(GFP_KERNEL, "%s-map", stem_name); + char *mask_name __free(kfree) = kasprintf(GFP_KERNEL, "%s-map-mask", stem_name); + char *pass_name __free(kfree) = kasprintf(GFP_KERNEL, "%s-map-pass-thru", stem_name); struct device_node *cur, *new = NULL; const __be32 *map, *mask, *pass; static const __be32 dummy_mask[] = { [0 ... MAX_PHANDLE_ARGS] = cpu_to_be32(~0) }; @@ -1407,27 +1410,13 @@ int of_parse_phandle_with_args_map(const struct device_node *np, if (index < 0) return -EINVAL; - cells_name = kasprintf(GFP_KERNEL, "#%s-cells", stem_name); - if (!cells_name) + if (!cells_name || !map_name || !mask_name || !pass_name) return -ENOMEM; - ret = -ENOMEM; - map_name = kasprintf(GFP_KERNEL, "%s-map", stem_name); - if (!map_name) - goto free; - - mask_name = kasprintf(GFP_KERNEL, "%s-map-mask", stem_name); - if (!mask_name) - goto free; - - pass_name = kasprintf(GFP_KERNEL, "%s-map-pass-thru", stem_name); - if (!pass_name) - goto free; - ret = __of_parse_phandle_with_args(np, list_name, cells_name, -1, index, out_args); if (ret) - goto free; + return ret; /* Get the #<list>-cells property */ cur = out_args->np; @@ -1444,8 +1433,7 @@ int of_parse_phandle_with_args_map(const struct device_node *np, /* Get the <list>-map property */ map = of_get_property(cur, map_name, &map_len); if (!map) { - ret = 0; - goto free; + return 0; } map_len /= sizeof(u32); @@ -1521,12 +1509,6 @@ int of_parse_phandle_with_args_map(const struct device_node *np, put: of_node_put(cur); of_node_put(new); -free: - kfree(mask_name); - kfree(map_name); - kfree(cells_name); - kfree(pass_name); - return ret; } EXPORT_SYMBOL(of_parse_phandle_with_args_map); diff --git a/drivers/of/device.c b/drivers/of/device.c index de89f9906375..312c63361211 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -95,8 +95,7 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, { const struct bus_dma_region *map = NULL; struct device_node *bus_np; - u64 dma_start = 0; - u64 mask, end, size = 0; + u64 mask, end = 0; bool coherent; int iommu_ret; int ret; @@ -117,34 +116,8 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, if (!force_dma) return ret == -ENODEV ? 0 : ret; } else { - const struct bus_dma_region *r = map; - u64 dma_end = 0; - /* Determine the overall bounds of all DMA regions */ - for (dma_start = ~0; r->size; r++) { - /* Take lower and upper limits */ - if (r->dma_start < dma_start) - dma_start = r->dma_start; - if (r->dma_start + r->size > dma_end) - dma_end = r->dma_start + r->size; - } - size = dma_end - dma_start; - - /* - * Add a work around to treat the size as mask + 1 in case - * it is defined in DT as a mask. - */ - if (size & 1) { - dev_warn(dev, "Invalid size 0x%llx for dma-range(s)\n", - size); - size = size + 1; - } - - if (!size) { - dev_err(dev, "Adjusted size 0x%llx invalid\n", size); - kfree(map); - return -EINVAL; - } + end = dma_range_map_max(map); } /* @@ -158,16 +131,15 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, dev->dma_mask = &dev->coherent_dma_mask; } - if (!size && dev->coherent_dma_mask) - size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1); - else if (!size) - size = 1ULL << 32; + if (!end && dev->coherent_dma_mask) + end = dev->coherent_dma_mask; + else if (!end) + end = (1ULL << 32) - 1; /* * Limit coherent and dma mask based on size and default mask * set by the driver. */ - end = dma_start + size - 1; mask = DMA_BIT_MASK(ilog2(end) + 1); dev->coherent_dma_mask &= mask; *dev->dma_mask &= mask; @@ -201,7 +173,7 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, } else dev_dbg(dev, "device is behind an iommu\n"); - arch_setup_dma_ops(dev, dma_start, size, coherent); + arch_setup_dma_ops(dev, coherent); if (iommu_ret) of_dma_set_restricted_buffer(dev, np); diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c index 4d57a4e34105..dda6092e6d3a 100644 --- a/drivers/of/dynamic.c +++ b/drivers/of/dynamic.c @@ -9,6 +9,7 @@ #define pr_fmt(fmt) "OF: " fmt +#include <linux/cleanup.h> #include <linux/device.h> #include <linux/of.h> #include <linux/spinlock.h> @@ -306,15 +307,20 @@ int of_detach_node(struct device_node *np) } EXPORT_SYMBOL_GPL(of_detach_node); +void __of_prop_free(struct property *prop) +{ + kfree(prop->name); + kfree(prop->value); + kfree(prop); +} + static void property_list_free(struct property *prop_list) { struct property *prop, *next; for (prop = prop_list; prop != NULL; prop = next) { next = prop->next; - kfree(prop->name); - kfree(prop->value); - kfree(prop); + __of_prop_free(prop); } } @@ -427,9 +433,7 @@ struct property *__of_prop_dup(const struct property *prop, gfp_t allocflags) return new; err_free: - kfree(new->name); - kfree(new->value); - kfree(new); + __of_prop_free(new); return NULL; } @@ -471,9 +475,7 @@ struct device_node *__of_node_dup(const struct device_node *np, if (!new_pp) goto err_prop; if (__of_add_property(node, new_pp)) { - kfree(new_pp->name); - kfree(new_pp->value); - kfree(new_pp); + __of_prop_free(new_pp); goto err_prop; } } @@ -933,11 +935,8 @@ static int of_changeset_add_prop_helper(struct of_changeset *ocs, return -ENOMEM; ret = of_changeset_add_property(ocs, np, new_pp); - if (ret) { - kfree(new_pp->name); - kfree(new_pp->value); - kfree(new_pp); - } + if (ret) + __of_prop_free(new_pp); return ret; } @@ -1033,10 +1032,9 @@ int of_changeset_add_prop_u32_array(struct of_changeset *ocs, const u32 *array, size_t sz) { struct property prop; - __be32 *val; - int i, ret; + __be32 *val __free(kfree) = kcalloc(sz, sizeof(__be32), GFP_KERNEL); + int i; - val = kcalloc(sz, sizeof(__be32), GFP_KERNEL); if (!val) return -ENOMEM; @@ -1046,9 +1044,6 @@ int of_changeset_add_prop_u32_array(struct of_changeset *ocs, prop.length = sizeof(u32) * sz; prop.value = (void *)val; - ret = of_changeset_add_prop_helper(ocs, np, &prop); - kfree(val); - - return ret; + return of_changeset_add_prop_helper(ocs, np, &prop); } EXPORT_SYMBOL_GPL(of_changeset_add_prop_u32_array); diff --git a/drivers/of/module.c b/drivers/of/module.c index f58e624953a2..780fd82a7ecc 100644 --- a/drivers/of/module.c +++ b/drivers/of/module.c @@ -29,14 +29,15 @@ ssize_t of_modalias(const struct device_node *np, char *str, ssize_t len) csize = snprintf(str, len, "of:N%pOFn%c%s", np, 'T', of_node_get_device_type(np)); tsize = csize; + if (csize >= len) + csize = len > 0 ? len - 1 : 0; len -= csize; - if (str) - str += csize; + str += csize; of_property_for_each_string(np, "compatible", p, compat) { csize = strlen(compat) + 1; tsize += csize; - if (csize > len) + if (csize >= len) continue; csize = snprintf(str, len, "C%s", compat); diff --git a/drivers/of/of_private.h b/drivers/of/of_private.h index 485483524b7f..94fc0aa07af9 100644 --- a/drivers/of/of_private.h +++ b/drivers/of/of_private.h @@ -123,6 +123,7 @@ extern void *__unflatten_device_tree(const void *blob, * own the devtree lock or work on detached trees only. */ struct property *__of_prop_dup(const struct property *prop, gfp_t allocflags); +void __of_prop_free(struct property *prop); struct device_node *__of_node_dup(const struct device_node *np, const char *full_name); diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c index 8236ecae2953..46e1c3fbc769 100644 --- a/drivers/of/of_reserved_mem.c +++ b/drivers/of/of_reserved_mem.c @@ -437,17 +437,10 @@ void __init fdt_init_reserved_mem(void) for (i = 0; i < reserved_mem_count; i++) { struct reserved_mem *rmem = &reserved_mem[i]; unsigned long node = rmem->fdt_node; - int len; - const __be32 *prop; int err = 0; bool nomap; nomap = of_get_flat_dt_prop(node, "no-map", NULL) != NULL; - prop = of_get_flat_dt_prop(node, "phandle", &len); - if (!prop) - prop = of_get_flat_dt_prop(node, "linux,phandle", &len); - if (prop) - rmem->phandle = of_read_number(prop, len/4); if (rmem->size == 0) err = __reserved_mem_alloc_size(node, rmem->name, @@ -477,19 +470,6 @@ void __init fdt_init_reserved_mem(void) } } -static inline struct reserved_mem *__find_rmem(struct device_node *node) -{ - unsigned int i; - - if (!node->phandle) - return NULL; - - for (i = 0; i < reserved_mem_count; i++) - if (reserved_mem[i].phandle == node->phandle) - return &reserved_mem[i]; - return NULL; -} - struct rmem_assigned_device { struct device *dev; struct reserved_mem *rmem; @@ -534,7 +514,7 @@ int of_reserved_mem_device_init_by_idx(struct device *dev, return 0; } - rmem = __find_rmem(target); + rmem = of_reserved_mem_lookup(target); of_node_put(target); if (!rmem || !rmem->ops || !rmem->ops->device_init) diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c index 2ae7e9d24a64..4d861a75d694 100644 --- a/drivers/of/overlay.c +++ b/drivers/of/overlay.c @@ -262,9 +262,7 @@ static struct property *dup_and_fixup_symbol_prop( return new_prop; err_free_new_prop: - kfree(new_prop->name); - kfree(new_prop->value); - kfree(new_prop); + __of_prop_free(new_prop); err_free_target_path: kfree(target_path); @@ -361,11 +359,8 @@ static int add_changeset_property(struct overlay_changeset *ovcs, pr_err("WARNING: memory leak will occur if overlay removed, property: %pOF/%s\n", target->np, new_prop->name); - if (ret) { - kfree(new_prop->name); - kfree(new_prop->value); - kfree(new_prop); - } + if (ret) + __of_prop_free(new_prop); return ret; } diff --git a/drivers/of/property.c b/drivers/of/property.c index 0320f1ae9b4d..1c83e68f805b 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -40,15 +40,12 @@ */ bool of_graph_is_present(const struct device_node *node) { - struct device_node *ports, *port; + struct device_node *ports __free(device_node) = of_get_child_by_name(node, "ports"); - ports = of_get_child_by_name(node, "ports"); if (ports) node = ports; - port = of_get_child_by_name(node, "port"); - of_node_put(ports); - of_node_put(port); + struct device_node *port __free(device_node) = of_get_child_by_name(node, "port"); return !!port; } @@ -579,7 +576,8 @@ EXPORT_SYMBOL_GPL(of_prop_next_string); int of_graph_parse_endpoint(const struct device_node *node, struct of_endpoint *endpoint) { - struct device_node *port_node = of_get_parent(node); + struct device_node *port_node __free(device_node) = + of_get_parent(node); WARN_ONCE(!port_node, "%s(): endpoint %pOF has no parent node\n", __func__, node); @@ -594,8 +592,6 @@ int of_graph_parse_endpoint(const struct device_node *node, of_property_read_u32(port_node, "reg", &endpoint->port); of_property_read_u32(node, "reg", &endpoint->id); - of_node_put(port_node); - return 0; } EXPORT_SYMBOL(of_graph_parse_endpoint); @@ -610,25 +606,22 @@ EXPORT_SYMBOL(of_graph_parse_endpoint); */ struct device_node *of_graph_get_port_by_id(struct device_node *parent, u32 id) { - struct device_node *node, *port; + struct device_node *node __free(device_node) = of_get_child_by_name(parent, "ports"); - node = of_get_child_by_name(parent, "ports"); if (node) parent = node; - for_each_child_of_node(parent, port) { + for_each_child_of_node_scoped(parent, port) { u32 port_id = 0; if (!of_node_name_eq(port, "port")) continue; of_property_read_u32(port, "reg", &port_id); if (id == port_id) - break; + return_ptr(port); } - of_node_put(node); - - return port; + return NULL; } EXPORT_SYMBOL(of_graph_get_port_by_id); @@ -655,15 +648,13 @@ struct device_node *of_graph_get_next_endpoint(const struct device_node *parent, * parent port node. */ if (!prev) { - struct device_node *node; + struct device_node *node __free(device_node) = + of_get_child_by_name(parent, "ports"); - node = of_get_child_by_name(parent, "ports"); if (node) parent = node; port = of_get_child_by_name(parent, "port"); - of_node_put(node); - if (!port) { pr_debug("graph: no port node found in %pOF\n", parent); return NULL; @@ -1052,15 +1043,13 @@ static int of_fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode, struct fwnode_endpoint *endpoint) { const struct device_node *node = to_of_node(fwnode); - struct device_node *port_node = of_get_parent(node); + struct device_node *port_node __free(device_node) = of_get_parent(node); endpoint->local_fwnode = fwnode; of_property_read_u32(port_node, "reg", &endpoint->port); of_property_read_u32(node, "reg", &endpoint->id); - of_node_put(port_node); - return 0; } @@ -1254,6 +1243,7 @@ DEFINE_SIMPLE_PROP(msi_parent, "msi-parent", "#msi-cells") DEFINE_SIMPLE_PROP(post_init_providers, "post-init-providers", NULL) DEFINE_SIMPLE_PROP(access_controllers, "access-controllers", "#access-controller-cells") DEFINE_SIMPLE_PROP(pses, "pses", "#pse-cells") +DEFINE_SIMPLE_PROP(power_supplies, "power-supplies", NULL) DEFINE_SUFFIX_PROP(regulators, "-supply", NULL) DEFINE_SUFFIX_PROP(gpio, "-gpio", "#gpio-cells") @@ -1313,6 +1303,57 @@ static struct device_node *parse_interrupts(struct device_node *np, return of_irq_parse_one(np, index, &sup_args) ? NULL : sup_args.np; } +static struct device_node *parse_interrupt_map(struct device_node *np, + const char *prop_name, int index) +{ + const __be32 *imap, *imap_end, *addr; + struct of_phandle_args sup_args; + u32 addrcells, intcells; + int i, imaplen; + + if (!IS_ENABLED(CONFIG_OF_IRQ)) + return NULL; + + if (strcmp(prop_name, "interrupt-map")) + return NULL; + + if (of_property_read_u32(np, "#interrupt-cells", &intcells)) + return NULL; + addrcells = of_bus_n_addr_cells(np); + + imap = of_get_property(np, "interrupt-map", &imaplen); + if (!imap || imaplen <= (addrcells + intcells)) + return NULL; + imap_end = imap + imaplen; + + while (imap < imap_end) { + addr = imap; + imap += addrcells; + + sup_args.np = np; + sup_args.args_count = intcells; + for (i = 0; i < intcells; i++) + sup_args.args[i] = be32_to_cpu(imap[i]); + imap += intcells; + + /* + * Upon success, the function of_irq_parse_raw() returns + * interrupt controller DT node pointer in sup_args.np. + */ + if (of_irq_parse_raw(addr, &sup_args)) + return NULL; + + if (!index) + return sup_args.np; + + of_node_put(sup_args.np); + imap += sup_args.args_count + 1; + index--; + } + + return NULL; +} + static struct device_node *parse_remote_endpoint(struct device_node *np, const char *prop_name, int index) @@ -1360,8 +1401,10 @@ static const struct supplier_bindings of_supplier_bindings[] = { { .parse_prop = parse_panel, }, { .parse_prop = parse_msi_parent, }, { .parse_prop = parse_pses, }, + { .parse_prop = parse_power_supplies, }, { .parse_prop = parse_gpio_compat, }, { .parse_prop = parse_interrupts, }, + { .parse_prop = parse_interrupt_map, }, { .parse_prop = parse_access_controllers, }, { .parse_prop = parse_regulators, }, { .parse_prop = parse_gpio, }, diff --git a/drivers/of/resolver.c b/drivers/of/resolver.c index b278ab4338ce..2780928764a4 100644 --- a/drivers/of/resolver.c +++ b/drivers/of/resolver.c @@ -8,6 +8,7 @@ #define pr_fmt(fmt) "OF: resolver: " fmt +#include <linux/cleanup.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> @@ -74,11 +75,11 @@ static int update_usages_of_a_phandle_reference(struct device_node *overlay, { struct device_node *refnode; struct property *prop; - char *value, *cur, *end, *node_path, *prop_name, *s; + char *value __free(kfree) = kmemdup(prop_fixup->value, prop_fixup->length, GFP_KERNEL); + char *cur, *end, *node_path, *prop_name, *s; int offset, len; int err = 0; - value = kmemdup(prop_fixup->value, prop_fixup->length, GFP_KERNEL); if (!value) return -ENOMEM; @@ -89,23 +90,19 @@ static int update_usages_of_a_phandle_reference(struct device_node *overlay, node_path = cur; s = strchr(cur, ':'); - if (!s) { - err = -EINVAL; - goto err_fail; - } + if (!s) + return -EINVAL; *s++ = '\0'; prop_name = s; s = strchr(s, ':'); - if (!s) { - err = -EINVAL; - goto err_fail; - } + if (!s) + return -EINVAL; *s++ = '\0'; err = kstrtoint(s, 10, &offset); if (err) - goto err_fail; + return err; refnode = __of_find_node_by_full_path(of_node_get(overlay), node_path); if (!refnode) @@ -117,22 +114,16 @@ static int update_usages_of_a_phandle_reference(struct device_node *overlay, } of_node_put(refnode); - if (!prop) { - err = -ENOENT; - goto err_fail; - } + if (!prop) + return -ENOENT; - if (offset < 0 || offset + sizeof(__be32) > prop->length) { - err = -EINVAL; - goto err_fail; - } + if (offset < 0 || offset + sizeof(__be32) > prop->length) + return -EINVAL; *(__be32 *)(prop->value + offset) = cpu_to_be32(phandle); } -err_fail: - kfree(value); - return err; + return 0; } /* compare nodes taking into account that 'name' strips out the @ part */ diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 6b5c36b6a758..445ad13dab98 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -795,15 +795,11 @@ static void __init of_unittest_property_copy(void) new = __of_prop_dup(&p1, GFP_KERNEL); unittest(new && propcmp(&p1, new), "empty property didn't copy correctly\n"); - kfree(new->value); - kfree(new->name); - kfree(new); + __of_prop_free(new); new = __of_prop_dup(&p2, GFP_KERNEL); unittest(new && propcmp(&p2, new), "non-empty property didn't copy correctly\n"); - kfree(new->value); - kfree(new->name); - kfree(new); + __of_prop_free(new); #endif } @@ -2815,7 +2811,7 @@ static int unittest_i2c_mux_probe(struct i2c_client *client) if (!muxc) return -ENOMEM; for (i = 0; i < nchans; i++) { - if (i2c_mux_add_adapter(muxc, 0, i, 0)) { + if (i2c_mux_add_adapter(muxc, 0, i)) { dev_err(dev, "Failed to register mux #%d\n", i); i2c_mux_del_adapters(muxc); return -ENODEV; @@ -3718,9 +3714,7 @@ static __init void of_unittest_overlay_high_level(void) goto err_unlock; } if (__of_add_property(of_symbols, new_prop)) { - kfree(new_prop->name); - kfree(new_prop->value); - kfree(new_prop); + __of_prop_free(new_prop); /* "name" auto-generated by unflatten */ if (!strcmp(prop->name, "name")) continue; diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c index 9ce0d20a6c58..feef537257d0 100644 --- a/drivers/parisc/ccio-dma.c +++ b/drivers/parisc/ccio-dma.c @@ -1022,7 +1022,7 @@ static const struct dma_map_ops ccio_ops = { .map_sg = ccio_map_sg, .unmap_sg = ccio_unmap_sg, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, }; diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c index 784037837f65..fc3863c09f83 100644 --- a/drivers/parisc/sba_iommu.c +++ b/drivers/parisc/sba_iommu.c @@ -1090,7 +1090,7 @@ static const struct dma_map_ops sba_ops = { .map_sg = sba_map_sg, .unmap_sg = sba_unmap_sg, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, }; diff --git a/drivers/pci/access.c b/drivers/pci/access.c index 6449056b57dd..30f031de9cfe 100644 --- a/drivers/pci/access.c +++ b/drivers/pci/access.c @@ -36,10 +36,13 @@ DEFINE_RAW_SPINLOCK(pci_lock); int noinline pci_bus_read_config_##size \ (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \ { \ - int res; \ unsigned long flags; \ u32 data = 0; \ - if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ + int res; \ + \ + if (PCI_##size##_BAD) \ + return PCIBIOS_BAD_REGISTER_NUMBER; \ + \ pci_lock_config(flags); \ res = bus->ops->read(bus, devfn, pos, len, &data); \ if (res) \ @@ -47,6 +50,7 @@ int noinline pci_bus_read_config_##size \ else \ *value = (type)data; \ pci_unlock_config(flags); \ + \ return res; \ } @@ -54,12 +58,16 @@ int noinline pci_bus_read_config_##size \ int noinline pci_bus_write_config_##size \ (struct pci_bus *bus, unsigned int devfn, int pos, type value) \ { \ - int res; \ unsigned long flags; \ - if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ + int res; \ + \ + if (PCI_##size##_BAD) \ + return PCIBIOS_BAD_REGISTER_NUMBER; \ + \ pci_lock_config(flags); \ res = bus->ops->write(bus, devfn, pos, len, value); \ pci_unlock_config(flags); \ + \ return res; \ } @@ -216,24 +224,27 @@ static noinline void pci_wait_cfg(struct pci_dev *dev) } /* Returns 0 on success, negative values indicate error. */ -#define PCI_USER_READ_CONFIG(size, type) \ +#define PCI_USER_READ_CONFIG(size, type) \ int pci_user_read_config_##size \ (struct pci_dev *dev, int pos, type *val) \ { \ - int ret = PCIBIOS_SUCCESSFUL; \ u32 data = -1; \ + int ret; \ + \ if (PCI_##size##_BAD) \ return -EINVAL; \ - raw_spin_lock_irq(&pci_lock); \ + \ + raw_spin_lock_irq(&pci_lock); \ if (unlikely(dev->block_cfg_access)) \ pci_wait_cfg(dev); \ ret = dev->bus->ops->read(dev->bus, dev->devfn, \ - pos, sizeof(type), &data); \ - raw_spin_unlock_irq(&pci_lock); \ + pos, sizeof(type), &data); \ + raw_spin_unlock_irq(&pci_lock); \ if (ret) \ PCI_SET_ERROR_RESPONSE(val); \ else \ *val = (type)data; \ + \ return pcibios_err_to_errno(ret); \ } \ EXPORT_SYMBOL_GPL(pci_user_read_config_##size); @@ -243,15 +254,18 @@ EXPORT_SYMBOL_GPL(pci_user_read_config_##size); int pci_user_write_config_##size \ (struct pci_dev *dev, int pos, type val) \ { \ - int ret = PCIBIOS_SUCCESSFUL; \ + int ret; \ + \ if (PCI_##size##_BAD) \ return -EINVAL; \ - raw_spin_lock_irq(&pci_lock); \ + \ + raw_spin_lock_irq(&pci_lock); \ if (unlikely(dev->block_cfg_access)) \ pci_wait_cfg(dev); \ ret = dev->bus->ops->write(dev->bus, dev->devfn, \ - pos, sizeof(type), val); \ - raw_spin_unlock_irq(&pci_lock); \ + pos, sizeof(type), val); \ + raw_spin_unlock_irq(&pci_lock); \ + \ return pcibios_err_to_errno(ret); \ } \ EXPORT_SYMBOL_GPL(pci_user_write_config_##size); @@ -275,6 +289,8 @@ void pci_cfg_access_lock(struct pci_dev *dev) { might_sleep(); + lock_map_acquire(&dev->cfg_access_lock); + raw_spin_lock_irq(&pci_lock); if (dev->block_cfg_access) pci_wait_cfg(dev); @@ -329,6 +345,8 @@ void pci_cfg_access_unlock(struct pci_dev *dev) raw_spin_unlock_irqrestore(&pci_lock, flags); wake_up_all(&pci_cfg_wait); + + lock_map_release(&dev->cfg_access_lock); } EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c index 81c50dc64da9..e0cc4560dfde 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-ep.c +++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c @@ -99,14 +99,11 @@ static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn, ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_IO_32BITS; } else { bool is_prefetch = !!(flags & PCI_BASE_ADDRESS_MEM_PREFETCH); - bool is_64bits = sz > SZ_2G; + bool is_64bits = !!(flags & PCI_BASE_ADDRESS_MEM_TYPE_64); if (is_64bits && (bar & 1)) return -EINVAL; - if (is_64bits && !(flags & PCI_BASE_ADDRESS_MEM_TYPE_64)) - epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64; - if (is_64bits && is_prefetch) ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_PREFETCH_MEM_64BITS; else if (is_prefetch) @@ -746,6 +743,8 @@ int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep) spin_lock_init(&ep->lock); + pci_epc_init_notify(epc); + return 0; free_epc_mem: diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c index 0e406677060d..d2d17d37d3e0 100644 --- a/drivers/pci/controller/dwc/pci-dra7xx.c +++ b/drivers/pci/controller/dwc/pci-dra7xx.c @@ -467,6 +467,15 @@ static int dra7xx_add_pcie_ep(struct dra7xx_pcie *dra7xx, return ret; } + ret = dw_pcie_ep_init_registers(ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(ep); + return ret; + } + + dw_pcie_ep_init_notify(ep); + return 0; } diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 99a60270b26c..917c69edee1d 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -1123,6 +1123,16 @@ static int imx6_add_pcie_ep(struct imx6_pcie *imx6_pcie, dev_err(dev, "failed to initialize endpoint\n"); return ret; } + + ret = dw_pcie_ep_init_registers(ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(ep); + return ret; + } + + dw_pcie_ep_init_notify(ep); + /* Start LTSSM. */ imx6_pcie_ltssm_enable(dev); diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index 844de4418724..d3a7d14ee685 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -1286,6 +1286,15 @@ static int ks_pcie_probe(struct platform_device *pdev) ret = dw_pcie_ep_init(&pci->ep); if (ret < 0) goto err_get_sync; + + ret = dw_pcie_ep_init_registers(&pci->ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + goto err_ep_init; + } + + dw_pcie_ep_init_notify(&pci->ep); + break; default: dev_err(dev, "INVALID device type %d\n", mode); @@ -1295,6 +1304,8 @@ static int ks_pcie_probe(struct platform_device *pdev) return 0; +err_ep_init: + dw_pcie_ep_deinit(&pci->ep); err_get_sync: pm_runtime_put(dev); pm_runtime_disable(dev); diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index 1f6ee1460ec2..7dde6d5fa4d8 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -279,6 +279,15 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) if (ret) return ret; + ret = dw_pcie_ep_init_registers(&pci->ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&pci->ep); + return ret; + } + + dw_pcie_ep_init_notify(&pci->ep); + return ls_pcie_ep_interrupt_init(pcie, pdev); } diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c index 9ed0a9ba7619..a4630b92489b 100644 --- a/drivers/pci/controller/dwc/pcie-artpec6.c +++ b/drivers/pci/controller/dwc/pcie-artpec6.c @@ -441,7 +441,20 @@ static int artpec6_pcie_probe(struct platform_device *pdev) pci->ep.ops = &pcie_ep_ops; - return dw_pcie_ep_init(&pci->ep); + ret = dw_pcie_ep_init(&pci->ep); + if (ret) + return ret; + + ret = dw_pcie_ep_init_registers(&pci->ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&pci->ep); + return ret; + } + + dw_pcie_ep_init_notify(&pci->ep); + + break; default: dev_err(dev, "INVALID device type %d\n", artpec6_pcie->mode); } diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 746a11dcb67f..47391d7d3a73 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -15,6 +15,10 @@ #include <linux/pci-epc.h> #include <linux/pci-epf.h> +/** + * dw_pcie_ep_linkup - Notify EPF drivers about Link Up event + * @ep: DWC EP device + */ void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) { struct pci_epc *epc = ep->epc; @@ -23,6 +27,10 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) } EXPORT_SYMBOL_GPL(dw_pcie_ep_linkup); +/** + * dw_pcie_ep_init_notify - Notify EPF drivers about EPC initialization complete + * @ep: DWC EP device + */ void dw_pcie_ep_init_notify(struct dw_pcie_ep *ep) { struct pci_epc *epc = ep->epc; @@ -31,6 +39,14 @@ void dw_pcie_ep_init_notify(struct dw_pcie_ep *ep) } EXPORT_SYMBOL_GPL(dw_pcie_ep_init_notify); +/** + * dw_pcie_ep_get_func_from_ep - Get the struct dw_pcie_ep_func corresponding to + * the endpoint function + * @ep: DWC EP device + * @func_no: Function number of the endpoint device + * + * Return: struct dw_pcie_ep_func if success, NULL otherwise. + */ struct dw_pcie_ep_func * dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) { @@ -61,6 +77,11 @@ static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, dw_pcie_dbi_ro_wr_dis(pci); } +/** + * dw_pcie_ep_reset_bar - Reset endpoint BAR + * @pci: DWC PCI device + * @bar: BAR number of the endpoint + */ void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) { u8 func_no, funcs; @@ -440,6 +461,13 @@ static const struct pci_epc_ops epc_ops = { .get_features = dw_pcie_ep_get_features, }; +/** + * dw_pcie_ep_raise_intx_irq - Raise INTx IRQ to the host + * @ep: DWC EP device + * @func_no: Function number of the endpoint + * + * Return: 0 if success, errono otherwise. + */ int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); @@ -451,6 +479,14 @@ int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no) } EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_intx_irq); +/** + * dw_pcie_ep_raise_msi_irq - Raise MSI IRQ to the host + * @ep: DWC EP device + * @func_no: Function number of the endpoint + * @interrupt_num: Interrupt number to be raised + * + * Return: 0 if success, errono otherwise. + */ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, u8 interrupt_num) { @@ -500,6 +536,15 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, } EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_msi_irq); +/** + * dw_pcie_ep_raise_msix_irq_doorbell - Raise MSI-X to the host using Doorbell + * method + * @ep: DWC EP device + * @func_no: Function number of the endpoint device + * @interrupt_num: Interrupt number to be raised + * + * Return: 0 if success, errno otherwise. + */ int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num) { @@ -519,6 +564,14 @@ int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, return 0; } +/** + * dw_pcie_ep_raise_msix_irq - Raise MSI-X to the host + * @ep: DWC EP device + * @func_no: Function number of the endpoint device + * @interrupt_num: Interrupt number to be raised + * + * Return: 0 if success, errno otherwise. + */ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num) { @@ -566,22 +619,42 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, return 0; } -void dw_pcie_ep_exit(struct dw_pcie_ep *ep) +/** + * dw_pcie_ep_cleanup - Cleanup DWC EP resources after fundamental reset + * @ep: DWC EP device + * + * Cleans up the DWC EP specific resources like eDMA etc... after fundamental + * reset like PERST#. Note that this API is only applicable for drivers + * supporting PERST# or any other methods of fundamental reset. + */ +void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); - struct pci_epc *epc = ep->epc; dw_pcie_edma_remove(pci); + ep->epc->init_complete = false; +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_cleanup); + +/** + * dw_pcie_ep_deinit - Deinitialize the endpoint device + * @ep: DWC EP device + * + * Deinitialize the endpoint device. EPC device is not destroyed since that will + * be taken care by Devres. + */ +void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) +{ + struct pci_epc *epc = ep->epc; + + dw_pcie_ep_cleanup(ep); pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem, epc->mem->window.page_size); pci_epc_mem_exit(epc); - - if (ep->ops->deinit) - ep->ops->deinit(ep); } -EXPORT_SYMBOL_GPL(dw_pcie_ep_exit); +EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit); static unsigned int dw_pcie_ep_find_ext_capability(struct dw_pcie *pci, int cap) { @@ -601,14 +674,27 @@ static unsigned int dw_pcie_ep_find_ext_capability(struct dw_pcie *pci, int cap) return 0; } -int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep) +/** + * dw_pcie_ep_init_registers - Initialize DWC EP specific registers + * @ep: DWC EP device + * + * Initialize the registers (CSRs) specific to DWC EP. This API should be called + * only when the endpoint receives an active refclk (either from host or + * generated locally). + */ +int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func; + struct device *dev = pci->dev; + struct pci_epc *epc = ep->epc; unsigned int offset, ptm_cap_base; unsigned int nbars; u8 hdr_type; + u8 func_no; + int i, ret; + void *addr; u32 reg; - int i; hdr_type = dw_pcie_readb_dbi(pci, PCI_HEADER_TYPE) & PCI_HEADER_TYPE_MASK; @@ -619,6 +705,58 @@ int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep) return -EIO; } + dw_pcie_version_detect(pci); + + dw_pcie_iatu_detect(pci); + + ret = dw_pcie_edma_detect(pci); + if (ret) + return ret; + + if (!ep->ib_window_map) { + ep->ib_window_map = devm_bitmap_zalloc(dev, pci->num_ib_windows, + GFP_KERNEL); + if (!ep->ib_window_map) + goto err_remove_edma; + } + + if (!ep->ob_window_map) { + ep->ob_window_map = devm_bitmap_zalloc(dev, pci->num_ob_windows, + GFP_KERNEL); + if (!ep->ob_window_map) + goto err_remove_edma; + } + + if (!ep->outbound_addr) { + addr = devm_kcalloc(dev, pci->num_ob_windows, sizeof(phys_addr_t), + GFP_KERNEL); + if (!addr) + goto err_remove_edma; + ep->outbound_addr = addr; + } + + for (func_no = 0; func_no < epc->max_functions; func_no++) { + + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (ep_func) + continue; + + ep_func = devm_kzalloc(dev, sizeof(*ep_func), GFP_KERNEL); + if (!ep_func) + goto err_remove_edma; + + ep_func->func_no = func_no; + ep_func->msi_cap = dw_pcie_ep_find_capability(ep, func_no, + PCI_CAP_ID_MSI); + ep_func->msix_cap = dw_pcie_ep_find_capability(ep, func_no, + PCI_CAP_ID_MSIX); + + list_add_tail(&ep_func->list, &ep->func_list); + } + + if (ep->ops->init) + ep->ops->init(ep); + offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); ptm_cap_base = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_PTM); @@ -658,22 +796,32 @@ int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep) dw_pcie_dbi_ro_wr_dis(pci); return 0; + +err_remove_edma: + dw_pcie_edma_remove(pci); + + return ret; } -EXPORT_SYMBOL_GPL(dw_pcie_ep_init_complete); +EXPORT_SYMBOL_GPL(dw_pcie_ep_init_registers); +/** + * dw_pcie_ep_init - Initialize the endpoint device + * @ep: DWC EP device + * + * Initialize the endpoint device. Allocate resources and create the EPC + * device with the endpoint framework. + * + * Return: 0 if success, errno otherwise. + */ int dw_pcie_ep_init(struct dw_pcie_ep *ep) { int ret; - void *addr; - u8 func_no; struct resource *res; struct pci_epc *epc; struct dw_pcie *pci = to_dw_pcie_from_ep(ep); struct device *dev = pci->dev; struct platform_device *pdev = to_platform_device(dev); struct device_node *np = dev->of_node; - const struct pci_epc_features *epc_features; - struct dw_pcie_ep_func *ep_func; INIT_LIST_HEAD(&ep->func_list); @@ -691,26 +839,6 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) if (ep->ops->pre_init) ep->ops->pre_init(ep); - dw_pcie_version_detect(pci); - - dw_pcie_iatu_detect(pci); - - ep->ib_window_map = devm_bitmap_zalloc(dev, pci->num_ib_windows, - GFP_KERNEL); - if (!ep->ib_window_map) - return -ENOMEM; - - ep->ob_window_map = devm_bitmap_zalloc(dev, pci->num_ob_windows, - GFP_KERNEL); - if (!ep->ob_window_map) - return -ENOMEM; - - addr = devm_kcalloc(dev, pci->num_ob_windows, sizeof(phys_addr_t), - GFP_KERNEL); - if (!addr) - return -ENOMEM; - ep->outbound_addr = addr; - epc = devm_pci_epc_create(dev, &epc_ops); if (IS_ERR(epc)) { dev_err(dev, "Failed to create epc device\n"); @@ -724,28 +852,11 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) if (ret < 0) epc->max_functions = 1; - for (func_no = 0; func_no < epc->max_functions; func_no++) { - ep_func = devm_kzalloc(dev, sizeof(*ep_func), GFP_KERNEL); - if (!ep_func) - return -ENOMEM; - - ep_func->func_no = func_no; - ep_func->msi_cap = dw_pcie_ep_find_capability(ep, func_no, - PCI_CAP_ID_MSI); - ep_func->msix_cap = dw_pcie_ep_find_capability(ep, func_no, - PCI_CAP_ID_MSIX); - - list_add_tail(&ep_func->list, &ep->func_list); - } - - if (ep->ops->init) - ep->ops->init(ep); - ret = pci_epc_mem_init(epc, ep->phys_base, ep->addr_size, ep->page_size); if (ret < 0) { dev_err(dev, "Failed to initialize address space\n"); - goto err_ep_deinit; + return ret; } ep->msi_mem = pci_epc_mem_alloc_addr(epc, &ep->msi_mem_phys, @@ -756,36 +867,11 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) goto err_exit_epc_mem; } - ret = dw_pcie_edma_detect(pci); - if (ret) - goto err_free_epc_mem; - - if (ep->ops->get_features) { - epc_features = ep->ops->get_features(ep); - if (epc_features->core_init_notifier) - return 0; - } - - ret = dw_pcie_ep_init_complete(ep); - if (ret) - goto err_remove_edma; - return 0; -err_remove_edma: - dw_pcie_edma_remove(pci); - -err_free_epc_mem: - pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem, - epc->mem->window.page_size); - err_exit_epc_mem: pci_epc_mem_exit(epc); -err_ep_deinit: - if (ep->ops->deinit) - ep->ops->deinit(ep); - return ret; } EXPORT_SYMBOL_GPL(dw_pcie_ep_init); diff --git a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c index 778588b4be70..8490c5d6ff9f 100644 --- a/drivers/pci/controller/dwc/pcie-designware-plat.c +++ b/drivers/pci/controller/dwc/pcie-designware-plat.c @@ -145,6 +145,17 @@ static int dw_plat_pcie_probe(struct platform_device *pdev) pci->ep.ops = &pcie_ep_ops; ret = dw_pcie_ep_init(&pci->ep); + if (ret) + return ret; + + ret = dw_pcie_ep_init_registers(&pci->ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&pci->ep); + } + + dw_pcie_ep_init_notify(&pci->ep); + break; default: dev_err(dev, "INVALID device type %d\n", dw_plat_pcie->mode); diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 26dae4837462..f8e5431a207b 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -333,7 +333,6 @@ struct dw_pcie_rp { struct dw_pcie_ep_ops { void (*pre_init)(struct dw_pcie_ep *ep); void (*init)(struct dw_pcie_ep *ep); - void (*deinit)(struct dw_pcie_ep *ep); int (*raise_irq)(struct dw_pcie_ep *ep, u8 func_no, unsigned int type, u16 interrupt_num); const struct pci_epc_features* (*get_features)(struct dw_pcie_ep *ep); @@ -670,9 +669,10 @@ static inline void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, #ifdef CONFIG_PCIE_DW_EP void dw_pcie_ep_linkup(struct dw_pcie_ep *ep); int dw_pcie_ep_init(struct dw_pcie_ep *ep); -int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep); +int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep); void dw_pcie_ep_init_notify(struct dw_pcie_ep *ep); -void dw_pcie_ep_exit(struct dw_pcie_ep *ep); +void dw_pcie_ep_deinit(struct dw_pcie_ep *ep); +void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep); int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no); int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, u8 interrupt_num); @@ -693,7 +693,7 @@ static inline int dw_pcie_ep_init(struct dw_pcie_ep *ep) return 0; } -static inline int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep) +static inline int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep) { return 0; } @@ -702,7 +702,11 @@ static inline void dw_pcie_ep_init_notify(struct dw_pcie_ep *ep) { } -static inline void dw_pcie_ep_exit(struct dw_pcie_ep *ep) +static inline void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) +{ +} + +static inline void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) { } diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c index 5e8e54f597dd..98bbc83182b4 100644 --- a/drivers/pci/controller/dwc/pcie-keembay.c +++ b/drivers/pci/controller/dwc/pcie-keembay.c @@ -396,6 +396,7 @@ static int keembay_pcie_probe(struct platform_device *pdev) struct keembay_pcie *pcie; struct dw_pcie *pci; enum dw_pcie_device_mode mode; + int ret; data = device_get_match_data(dev); if (!data) @@ -430,11 +431,26 @@ static int keembay_pcie_probe(struct platform_device *pdev) return -ENODEV; pci->ep.ops = &keembay_pcie_ep_ops; - return dw_pcie_ep_init(&pci->ep); + ret = dw_pcie_ep_init(&pci->ep); + if (ret) + return ret; + + ret = dw_pcie_ep_init_registers(&pci->ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&pci->ep); + return ret; + } + + dw_pcie_ep_init_notify(&pci->ep); + + break; default: dev_err(dev, "Invalid device type %d\n", pcie->mode); return -ENODEV; } + + return 0; } static const struct keembay_pcie_of_data keembay_pcie_rc_of_data = { diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index 36e5e80cd22f..2fb8c15e7a91 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -463,7 +463,7 @@ static int qcom_pcie_perst_deassert(struct dw_pcie *pci) PARF_INT_ALL_LINK_UP | PARF_INT_ALL_EDMA; writel_relaxed(val, pcie_ep->parf + PARF_INT_ALL_MASK); - ret = dw_pcie_ep_init_complete(&pcie_ep->pci.ep); + ret = dw_pcie_ep_init_registers(&pcie_ep->pci.ep); if (ret) { dev_err(dev, "Failed to complete initialization: %d\n", ret); goto err_disable_resources; @@ -507,6 +507,7 @@ static void qcom_pcie_perst_assert(struct dw_pcie *pci) return; } + dw_pcie_ep_cleanup(&pci->ep); qcom_pcie_disable_resources(pcie_ep); pcie_ep->link_status = QCOM_PCIE_EP_LINK_DISABLED; } @@ -774,7 +775,6 @@ static void qcom_pcie_ep_init_debugfs(struct qcom_pcie_ep *pcie_ep) static const struct pci_epc_features qcom_pcie_epc_features = { .linkup_notifier = true, - .core_init_notifier = true, .msi_capable = true, .msix_capable = false, .align = SZ_4K, diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c index 0be760ed420b..cfeccc2f9ee1 100644 --- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c +++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c @@ -352,11 +352,8 @@ static void rcar_gen4_pcie_ep_init(struct dw_pcie_ep *ep) dw_pcie_ep_reset_bar(pci, bar); } -static void rcar_gen4_pcie_ep_deinit(struct dw_pcie_ep *ep) +static void rcar_gen4_pcie_ep_deinit(struct rcar_gen4_pcie *rcar) { - struct dw_pcie *dw = to_dw_pcie_from_ep(ep); - struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw); - writel(0, rcar->base + PCIEDMAINTSTSEN); rcar_gen4_pcie_common_deinit(rcar); } @@ -410,7 +407,6 @@ static unsigned int rcar_gen4_pcie_ep_get_dbi2_offset(struct dw_pcie_ep *ep, static const struct dw_pcie_ep_ops pcie_ep_ops = { .pre_init = rcar_gen4_pcie_ep_pre_init, .init = rcar_gen4_pcie_ep_init, - .deinit = rcar_gen4_pcie_ep_deinit, .raise_irq = rcar_gen4_pcie_ep_raise_irq, .get_features = rcar_gen4_pcie_ep_get_features, .get_dbi_offset = rcar_gen4_pcie_ep_get_dbi_offset, @@ -420,18 +416,36 @@ static const struct dw_pcie_ep_ops pcie_ep_ops = { static int rcar_gen4_add_dw_pcie_ep(struct rcar_gen4_pcie *rcar) { struct dw_pcie_ep *ep = &rcar->dw.ep; + struct device *dev = rcar->dw.dev; + int ret; if (!IS_ENABLED(CONFIG_PCIE_RCAR_GEN4_EP)) return -ENODEV; ep->ops = &pcie_ep_ops; - return dw_pcie_ep_init(ep); + ret = dw_pcie_ep_init(ep); + if (ret) { + rcar_gen4_pcie_ep_deinit(rcar); + return ret; + } + + ret = dw_pcie_ep_init_registers(ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(ep); + rcar_gen4_pcie_ep_deinit(rcar); + } + + dw_pcie_ep_init_notify(ep); + + return ret; } static void rcar_gen4_remove_dw_pcie_ep(struct rcar_gen4_pcie *rcar) { - dw_pcie_ep_exit(&rcar->dw.ep); + dw_pcie_ep_deinit(&rcar->dw.ep); + rcar_gen4_pcie_ep_deinit(rcar); } /* Common */ diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 1f7b662cb8e1..93f5433c5c55 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -1715,6 +1715,8 @@ static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) if (ret) dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret); + dw_pcie_ep_cleanup(&pcie->pci.ep); + reset_control_assert(pcie->core_rst); tegra_pcie_disable_phy(pcie); @@ -1895,7 +1897,7 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val); - ret = dw_pcie_ep_init_complete(ep); + ret = dw_pcie_ep_init_registers(ep); if (ret) { dev_err(dev, "Failed to complete initialization: %d\n", ret); goto fail_init_complete; @@ -2004,7 +2006,6 @@ static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, static const struct pci_epc_features tegra_pcie_epc_features = { .linkup_notifier = true, - .core_init_notifier = true, .msi_capable = false, .msix_capable = false, .bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M, @@ -2273,11 +2274,14 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev) ret = tegra_pcie_config_ep(pcie, pdev); if (ret < 0) goto fail; + else + return 0; break; default: dev_err(dev, "Invalid PCIe device type %d\n", pcie->of_data->mode); + ret = -EINVAL; } fail: diff --git a/drivers/pci/controller/dwc/pcie-uniphier-ep.c b/drivers/pci/controller/dwc/pcie-uniphier-ep.c index 639bc2e12476..a2b844268e28 100644 --- a/drivers/pci/controller/dwc/pcie-uniphier-ep.c +++ b/drivers/pci/controller/dwc/pcie-uniphier-ep.c @@ -399,7 +399,20 @@ static int uniphier_pcie_ep_probe(struct platform_device *pdev) return ret; priv->pci.ep.ops = &uniphier_pcie_ep_ops; - return dw_pcie_ep_init(&priv->pci.ep); + ret = dw_pcie_ep_init(&priv->pci.ep); + if (ret) + return ret; + + ret = dw_pcie_ep_init_registers(&priv->pci.ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&priv->pci.ep); + return ret; + } + + dw_pcie_ep_init_notify(&priv->pci.ep); + + return 0; } static const struct uniphier_pcie_ep_soc_data uniphier_pro5_data = { diff --git a/drivers/pci/controller/pcie-mt7621.c b/drivers/pci/controller/pcie-mt7621.c index 79e225edb42a..d97b956e6e57 100644 --- a/drivers/pci/controller/pcie-mt7621.c +++ b/drivers/pci/controller/pcie-mt7621.c @@ -202,7 +202,7 @@ static int mt7621_pcie_parse_port(struct mt7621_pcie *pcie, struct mt7621_pcie_port *port; struct device *dev = pcie->dev; struct platform_device *pdev = to_platform_device(dev); - char name[10]; + char name[11]; int err; port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); diff --git a/drivers/pci/controller/pcie-rcar-ep.c b/drivers/pci/controller/pcie-rcar-ep.c index 05967c6c0b42..047e2cef5afc 100644 --- a/drivers/pci/controller/pcie-rcar-ep.c +++ b/drivers/pci/controller/pcie-rcar-ep.c @@ -542,6 +542,8 @@ static int rcar_pcie_ep_probe(struct platform_device *pdev) goto err_pm_put; } + pci_epc_init_notify(epc); + return 0; err_pm_put: diff --git a/drivers/pci/controller/pcie-rockchip-ep.c b/drivers/pci/controller/pcie-rockchip-ep.c index c9046e97a1d2..136274533656 100644 --- a/drivers/pci/controller/pcie-rockchip-ep.c +++ b/drivers/pci/controller/pcie-rockchip-ep.c @@ -98,10 +98,8 @@ static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, /* All functions share the same vendor ID with function 0 */ if (fn == 0) { - u32 vid_regs = (hdr->vendorid & GENMASK(15, 0)) | - (hdr->subsys_vendor_id & GENMASK(31, 16)) << 16; - - rockchip_pcie_write(rockchip, vid_regs, + rockchip_pcie_write(rockchip, + hdr->vendorid | hdr->subsys_vendor_id << 16, PCIE_CORE_CONFIG_VENDOR); } @@ -153,7 +151,7 @@ static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn, ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_IO_32BITS; } else { bool is_prefetch = !!(flags & PCI_BASE_ADDRESS_MEM_PREFETCH); - bool is_64bits = sz > SZ_2G; + bool is_64bits = !!(flags & PCI_BASE_ADDRESS_MEM_TYPE_64); if (is_64bits && (bar & 1)) return -EINVAL; @@ -609,6 +607,8 @@ static int rockchip_pcie_ep_probe(struct platform_device *pdev) rockchip_pcie_write(rockchip, PCIE_CLIENT_CONF_ENABLE, PCIE_CLIENT_CONFIG); + pci_epc_init_notify(epc); + return 0; err_epc_mem_exit: pci_epc_mem_exit(epc); diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c index e3aab5edaf70..652d63df9d22 100644 --- a/drivers/pci/doe.c +++ b/drivers/pci/doe.c @@ -383,11 +383,13 @@ static void pci_doe_task_complete(struct pci_doe_task *task) complete(task->private); } -static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid, +static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 capver, u8 *index, u16 *vid, u8 *protocol) { u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX, - *index); + *index) | + FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_VER, + (capver >= 2) ? 2 : 0); __le32 request_pl_le = cpu_to_le32(request_pl); __le32 response_pl_le; u32 response_pl; @@ -421,13 +423,17 @@ static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb) { u8 index = 0; u8 xa_idx = 0; + u32 hdr = 0; + + pci_read_config_dword(doe_mb->pdev, doe_mb->cap_offset, &hdr); do { int rc; u16 vid; u8 prot; - rc = pci_doe_discovery(doe_mb, &index, &vid, &prot); + rc = pci_doe_discovery(doe_mb, PCI_EXT_CAP_VER(hdr), &index, + &vid, &prot); if (rc) return rc; diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index cd4ffb39dcdc..977fb79c1567 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -690,50 +690,35 @@ static void pci_epf_test_unbind(struct pci_epf *epf) { struct pci_epf_test *epf_test = epf_get_drvdata(epf); struct pci_epc *epc = epf->epc; - struct pci_epf_bar *epf_bar; int bar; cancel_delayed_work(&epf_test->cmd_handler); pci_epf_test_clean_dma_chan(epf_test); for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { - epf_bar = &epf->bar[bar]; + if (!epf_test->reg[bar]) + continue; - if (epf_test->reg[bar]) { - pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, - epf_bar); - pci_epf_free_space(epf, epf_test->reg[bar], bar, - PRIMARY_INTERFACE); - } + pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, + &epf->bar[bar]); + pci_epf_free_space(epf, epf_test->reg[bar], bar, + PRIMARY_INTERFACE); } } static int pci_epf_test_set_bar(struct pci_epf *epf) { - int bar, add; - int ret; - struct pci_epf_bar *epf_bar; + int bar, ret; struct pci_epc *epc = epf->epc; struct device *dev = &epf->dev; struct pci_epf_test *epf_test = epf_get_drvdata(epf); enum pci_barno test_reg_bar = epf_test->test_reg_bar; - const struct pci_epc_features *epc_features; - - epc_features = epf_test->epc_features; - for (bar = 0; bar < PCI_STD_NUM_BARS; bar += add) { - epf_bar = &epf->bar[bar]; - /* - * pci_epc_set_bar() sets PCI_BASE_ADDRESS_MEM_TYPE_64 - * if the specific implementation required a 64-bit BAR, - * even if we only requested a 32-bit BAR. - */ - add = (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ? 2 : 1; - - if (epc_features->bar[bar].type == BAR_RESERVED) + for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { + if (!epf_test->reg[bar]) continue; ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, - epf_bar); + &epf->bar[bar]); if (ret) { pci_epf_free_space(epf, epf_test->reg[bar], bar, PRIMARY_INTERFACE); @@ -753,6 +738,7 @@ static int pci_epf_test_core_init(struct pci_epf *epf) const struct pci_epc_features *epc_features; struct pci_epc *epc = epf->epc; struct device *dev = &epf->dev; + bool linkup_notifier = false; bool msix_capable = false; bool msi_capable = true; int ret; @@ -795,6 +781,10 @@ static int pci_epf_test_core_init(struct pci_epf *epf) } } + linkup_notifier = epc_features->linkup_notifier; + if (!linkup_notifier) + queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work); + return 0; } @@ -817,14 +807,13 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) { struct pci_epf_test *epf_test = epf_get_drvdata(epf); struct device *dev = &epf->dev; - struct pci_epf_bar *epf_bar; size_t msix_table_size = 0; size_t test_reg_bar_size; size_t pba_size = 0; bool msix_capable; void *base; - int bar, add; enum pci_barno test_reg_bar = epf_test->test_reg_bar; + enum pci_barno bar; const struct pci_epc_features *epc_features; size_t test_reg_size; @@ -849,16 +838,14 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) } epf_test->reg[test_reg_bar] = base; - for (bar = 0; bar < PCI_STD_NUM_BARS; bar += add) { - epf_bar = &epf->bar[bar]; - add = (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ? 2 : 1; + for (bar = BAR_0; bar < PCI_STD_NUM_BARS; bar++) { + bar = pci_epc_get_next_free_bar(epc_features, bar); + if (bar == NO_BAR) + break; if (bar == test_reg_bar) continue; - if (epc_features->bar[bar].type == BAR_RESERVED) - continue; - base = pci_epf_alloc_space(epf, bar_size[bar], bar, epc_features, PRIMARY_INTERFACE); if (!base) @@ -870,19 +857,6 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) return 0; } -static void pci_epf_configure_bar(struct pci_epf *epf, - const struct pci_epc_features *epc_features) -{ - struct pci_epf_bar *epf_bar; - int i; - - for (i = 0; i < PCI_STD_NUM_BARS; i++) { - epf_bar = &epf->bar[i]; - if (epc_features->bar[i].only_64bit) - epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64; - } -} - static int pci_epf_test_bind(struct pci_epf *epf) { int ret; @@ -890,8 +864,6 @@ static int pci_epf_test_bind(struct pci_epf *epf) const struct pci_epc_features *epc_features; enum pci_barno test_reg_bar = BAR_0; struct pci_epc *epc = epf->epc; - bool linkup_notifier = false; - bool core_init_notifier = false; if (WARN_ON_ONCE(!epc)) return -EINVAL; @@ -902,12 +874,9 @@ static int pci_epf_test_bind(struct pci_epf *epf) return -EOPNOTSUPP; } - linkup_notifier = epc_features->linkup_notifier; - core_init_notifier = epc_features->core_init_notifier; test_reg_bar = pci_epc_get_first_free_bar(epc_features); if (test_reg_bar < 0) return -EINVAL; - pci_epf_configure_bar(epf, epc_features); epf_test->test_reg_bar = test_reg_bar; epf_test->epc_features = epc_features; @@ -916,21 +885,12 @@ static int pci_epf_test_bind(struct pci_epf *epf) if (ret) return ret; - if (!core_init_notifier) { - ret = pci_epf_test_core_init(epf); - if (ret) - return ret; - } - epf_test->dma_supported = true; ret = pci_epf_test_init_dma_chan(epf_test); if (ret) epf_test->dma_supported = false; - if (!linkup_notifier && !core_init_notifier) - queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work); - return 0; } diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index 0ea64e24ed61..3b21e28f9b59 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -64,6 +64,9 @@ static int pci_secondary_epc_epf_link(struct config_item *epf_item, return ret; } + /* Send any pending EPC initialization complete to the EPF driver */ + pci_epc_notify_pending_init(epc, epf); + return 0; } @@ -125,6 +128,9 @@ static int pci_primary_epc_epf_link(struct config_item *epf_item, return ret; } + /* Send any pending EPC initialization complete to the EPF driver */ + pci_epc_notify_pending_init(epc, epf); + return 0; } @@ -230,6 +236,9 @@ static int pci_epc_epf_link(struct config_item *epc_item, return ret; } + /* Send any pending EPC initialization complete to the EPF driver */ + pci_epc_notify_pending_init(epc, epf); + return 0; } diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index da3fc0795b0b..47d27ec7439d 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -748,11 +748,33 @@ void pci_epc_init_notify(struct pci_epc *epc) epf->event_ops->core_init(epf); mutex_unlock(&epf->lock); } + epc->init_complete = true; mutex_unlock(&epc->list_lock); } EXPORT_SYMBOL_GPL(pci_epc_init_notify); /** + * pci_epc_notify_pending_init() - Notify the pending EPC device initialization + * complete to the EPF device + * @epc: the EPC device whose core initialization is pending to be notified + * @epf: the EPF device to be notified + * + * Invoke to notify the pending EPC device initialization complete to the EPF + * device. This is used to deliver the notification if the EPC initialization + * got completed before the EPF driver bind. + */ +void pci_epc_notify_pending_init(struct pci_epc *epc, struct pci_epf *epf) +{ + if (epc->init_complete) { + mutex_lock(&epf->lock); + if (epf->event_ops && epf->event_ops->core_init) + epf->event_ops->core_init(epf); + mutex_unlock(&epf->lock); + } +} +EXPORT_SYMBOL_GPL(pci_epc_notify_pending_init); + +/** * pci_epc_bme_notify() - Notify the EPF device that the EPC device has received * the BME event from the Root complex * @epc: the EPC device that received the BME event diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index 0a28a0b0911b..323f2a60ab16 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -255,6 +255,8 @@ EXPORT_SYMBOL_GPL(pci_epf_free_space); * @type: Identifies if the allocation is for primary EPC or secondary EPC * * Invoke to allocate memory for the PCI EPF register space. + * Flag PCI_BASE_ADDRESS_MEM_TYPE_64 will automatically get set if the BAR + * can only be a 64-bit BAR, or if the requested size is larger than 2 GB. */ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, const struct pci_epc_features *epc_features, @@ -304,9 +306,10 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, epf_bar[bar].addr = space; epf_bar[bar].size = size; epf_bar[bar].barno = bar; - epf_bar[bar].flags |= upper_32_bits(size) ? - PCI_BASE_ADDRESS_MEM_TYPE_64 : - PCI_BASE_ADDRESS_MEM_TYPE_32; + if (upper_32_bits(size) || epc_features->bar[bar].only_64bit) + epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64; + else + epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_32; return space; } diff --git a/drivers/pci/hotplug/TODO b/drivers/pci/hotplug/TODO index fdb8dd6ea24d..9d428b0ea524 100644 --- a/drivers/pci/hotplug/TODO +++ b/drivers/pci/hotplug/TODO @@ -6,6 +6,8 @@ cpcihp: ->set_power callbacks in struct cpci_hp_controller_ops. Why were they introduced? Can they be removed from the struct? +* Returned code from pci_hp_add_bridge() is not checked. + cpqphp: * The driver spawns a kthread cpqhp_event_thread() which is woken by the @@ -16,6 +18,8 @@ cpqphp: * A large portion of cpqphp_ctrl.c and cpqphp_pci.c concerns resource management. Doesn't this duplicate functionality in the core? +* Returned code from pci_hp_add_bridge() is not checked. + ibmphp: * Implementations of hotplug_slot_ops callbacks such as get_adapter_present() @@ -43,13 +47,7 @@ ibmphp: * A large portion of ibmphp_res.c and ibmphp_pci.c concerns resource management. Doesn't this duplicate functionality in the core? -sgi_hotplug: - -* Several functions access the pci_slot member in struct hotplug_slot even - though pci_hotplug.h declares it private. See sn_hp_destroy() for an - example. Either the pci_slot member should no longer be declared private - or sgi_hotplug should store a pointer to it in struct slot. Probably the - former. +* Returned code from pci_hp_add_bridge() is not checked. shpchp: diff --git a/drivers/pci/msi/api.c b/drivers/pci/msi/api.c index be679aa5db64..b956ce591f96 100644 --- a/drivers/pci/msi/api.c +++ b/drivers/pci/msi/api.c @@ -213,8 +213,8 @@ EXPORT_SYMBOL(pci_disable_msix); * * %PCI_IRQ_MSIX Allow trying MSI-X vector allocations * * %PCI_IRQ_MSI Allow trying MSI vector allocations * - * * %PCI_IRQ_LEGACY Allow trying legacy INTx interrupts, if - * and only if @min_vecs == 1 + * * %PCI_IRQ_INTX Allow trying INTx interrupts, if and + * only if @min_vecs == 1 * * * %PCI_IRQ_AFFINITY Auto-manage IRQs affinity by spreading * the vectors around available CPUs @@ -279,8 +279,8 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs, return nvecs; } - /* use legacy IRQ if allowed */ - if (flags & PCI_IRQ_LEGACY) { + /* use INTx IRQ if allowed */ + if (flags & PCI_IRQ_INTX) { if (min_vecs == 1 && dev->irq) { /* * Invoke the affinity spreading logic to ensure that @@ -366,56 +366,6 @@ const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr) EXPORT_SYMBOL(pci_irq_get_affinity); /** - * pci_ims_alloc_irq - Allocate an interrupt on a PCI/IMS interrupt domain - * @dev: The PCI device to operate on - * @icookie: Pointer to an IMS implementation specific cookie for this - * IMS instance (PASID, queue ID, pointer...). - * The cookie content is copied into the MSI descriptor for the - * interrupt chip callbacks or domain specific setup functions. - * @affdesc: Optional pointer to an interrupt affinity descriptor - * - * There is no index for IMS allocations as IMS is an implementation - * specific storage and does not have any direct associations between - * index, which might be a pure software construct, and device - * functionality. This association is established by the driver either via - * the index - if there is a hardware table - or in case of purely software - * managed IMS implementation the association happens via the - * irq_write_msi_msg() callback of the implementation specific interrupt - * chip, which utilizes the provided @icookie to store the MSI message in - * the appropriate place. - * - * Return: A struct msi_map - * - * On success msi_map::index contains the allocated index (>= 0) and - * msi_map::virq the allocated Linux interrupt number (> 0). - * - * On fail msi_map::index contains the error code and msi_map::virq - * is set to 0. - */ -struct msi_map pci_ims_alloc_irq(struct pci_dev *dev, union msi_instance_cookie *icookie, - const struct irq_affinity_desc *affdesc) -{ - return msi_domain_alloc_irq_at(&dev->dev, MSI_SECONDARY_DOMAIN, MSI_ANY_INDEX, - affdesc, icookie); -} -EXPORT_SYMBOL_GPL(pci_ims_alloc_irq); - -/** - * pci_ims_free_irq - Allocate an interrupt on a PCI/IMS interrupt domain - * which was allocated via pci_ims_alloc_irq() - * @dev: The PCI device to operate on - * @map: A struct msi_map describing the interrupt to free as - * returned from pci_ims_alloc_irq() - */ -void pci_ims_free_irq(struct pci_dev *dev, struct msi_map map) -{ - if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0)) - return; - msi_domain_free_irqs_range(&dev->dev, MSI_SECONDARY_DOMAIN, map.index, map.index); -} -EXPORT_SYMBOL_GPL(pci_ims_free_irq); - -/** * pci_free_irq_vectors() - Free previously allocated IRQs for a device * @dev: the PCI device to operate on * diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index cfd84a899c82..03d2dd25790d 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -355,65 +355,6 @@ bool pci_msi_domain_supports(struct pci_dev *pdev, unsigned int feature_mask, return (supported & feature_mask) == feature_mask; } -/** - * pci_create_ims_domain - Create a secondary IMS domain for a PCI device - * @pdev: The PCI device to operate on - * @template: The MSI info template which describes the domain - * @hwsize: The size of the hardware entry table or 0 if the domain - * is purely software managed - * @data: Optional pointer to domain specific data to be stored - * in msi_domain_info::data - * - * Return: True on success, false otherwise - * - * An IMS domain is expected to have the following constraints: - * - The index space is managed by the core code - * - * - There is no requirement for consecutive index ranges - * - * - The interrupt chip must provide the following callbacks: - * - irq_mask() - * - irq_unmask() - * - irq_write_msi_msg() - * - * - The interrupt chip must provide the following optional callbacks - * when the irq_mask(), irq_unmask() and irq_write_msi_msg() callbacks - * cannot operate directly on hardware, e.g. in the case that the - * interrupt message store is in queue memory: - * - irq_bus_lock() - * - irq_bus_unlock() - * - * These callbacks are invoked from preemptible task context and are - * allowed to sleep. In this case the mandatory callbacks above just - * store the information. The irq_bus_unlock() callback is supposed - * to make the change effective before returning. - * - * - Interrupt affinity setting is handled by the underlying parent - * interrupt domain and communicated to the IMS domain via - * irq_write_msi_msg(). - * - * The domain is automatically destroyed when the PCI device is removed. - */ -bool pci_create_ims_domain(struct pci_dev *pdev, const struct msi_domain_template *template, - unsigned int hwsize, void *data) -{ - struct irq_domain *domain = dev_get_msi_domain(&pdev->dev); - - if (!domain || !irq_domain_is_msi_parent(domain)) - return false; - - if (template->info.bus_token != DOMAIN_BUS_PCI_DEVICE_IMS || - !(template->info.flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS) || - !(template->info.flags & MSI_FLAG_FREE_MSI_DESCS) || - !template->chip.irq_mask || !template->chip.irq_unmask || - !template->chip.irq_write_msi_msg || template->chip.irq_set_affinity) - return false; - - return msi_create_device_irq_domain(&pdev->dev, MSI_SECONDARY_DOMAIN, template, - hwsize, data, NULL); -} -EXPORT_SYMBOL_GPL(pci_create_ims_domain); - /* * Users of the generic MSI infrastructure expect a device to have a single ID, * so with DMA aliases we have to pick the least-worst compromise. Devices with diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 682fa877478f..c5625dd9bf49 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -86,9 +86,11 @@ static int pcim_setup_msi_release(struct pci_dev *dev) return 0; ret = devm_add_action(&dev->dev, pcim_msi_release, dev); - if (!ret) - dev->is_msi_managed = true; - return ret; + if (ret) + return ret; + + dev->is_msi_managed = true; + return 0; } /* @@ -99,9 +101,10 @@ static int pci_setup_msi_context(struct pci_dev *dev) { int ret = msi_setup_device_data(&dev->dev); - if (!ret) - ret = pcim_setup_msi_release(dev); - return ret; + if (ret) + return ret; + + return pcim_setup_msi_release(dev); } /* diff --git a/drivers/pci/of_property.c b/drivers/pci/of_property.c index c2c7334152bc..03539e505372 100644 --- a/drivers/pci/of_property.c +++ b/drivers/pci/of_property.c @@ -238,6 +238,8 @@ static int of_pci_prop_intr_map(struct pci_dev *pdev, struct of_changeset *ocs, return 0; int_map = kcalloc(map_sz, sizeof(u32), GFP_KERNEL); + if (!int_map) + return -ENOMEM; mapp = int_map; list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4288..59e0949fb079 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -142,8 +142,8 @@ enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT; * the dfl or actual value as it sees fit. Don't forget this is * measured in 32-bit words, not bytes. */ -u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2; -u8 pci_cache_line_size; +u8 pci_dfl_cache_line_size __ro_after_init = L1_CACHE_BYTES >> 2; +u8 pci_cache_line_size __ro_after_init ; /* * If we set up a device for bus mastering, we need to check the latency @@ -1277,6 +1277,11 @@ static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) for (;;) { u32 id; + if (pci_dev_is_disconnected(dev)) { + pci_dbg(dev, "disconnected; not waiting\n"); + return -ENOTTY; + } + pci_read_config_dword(dev, PCI_COMMAND, &id); if (!PCI_POSSIBLE_ERROR(id)) break; @@ -2111,20 +2116,6 @@ static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags) } /** - * pci_enable_device_io - Initialize a device for use with IO space - * @dev: PCI device to be initialized - * - * Initialize device before it's used by a driver. Ask low-level code - * to enable I/O resources. Wake up the device if it was suspended. - * Beware, this function can fail. - */ -int pci_enable_device_io(struct pci_dev *dev) -{ - return pci_enable_device_flags(dev, IORESOURCE_IO); -} -EXPORT_SYMBOL(pci_enable_device_io); - -/** * pci_enable_device_mem - Initialize a device for use with Memory space * @dev: PCI device to be initialized * @@ -2962,6 +2953,18 @@ static const struct dmi_system_id bridge_d3_blacklist[] = { DMI_MATCH(DMI_BOARD_VERSION, "Continental Z2"), }, }, + { + /* + * Changing power state of root port dGPU is connected fails + * https://gitlab.freedesktop.org/drm/amd/-/issues/3229 + */ + .ident = "Hewlett-Packard HP Pavilion 17 Notebook PC/1972", + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Hewlett-Packard"), + DMI_MATCH(DMI_BOARD_NAME, "1972"), + DMI_MATCH(DMI_BOARD_VERSION, "95.33"), + }, + }, #endif { } }; @@ -4625,11 +4628,12 @@ int pcie_retrain_link(struct pci_dev *pdev, bool use_lt) /* * Ensure the updated LNKCTL parameters are used during link - * training by checking that there is no ongoing link training to - * avoid LTSSM race as recommended in Implementation Note at the - * end of PCIe r6.0.1 sec 7.5.3.7. + * training by checking that there is no ongoing link training that + * may have started before link parameters were changed, so as to + * avoid LTSSM race as recommended in Implementation Note at the end + * of PCIe r6.1 sec 7.5.3.7. */ - rc = pcie_wait_for_link_status(pdev, use_lt, !use_lt); + rc = pcie_wait_for_link_status(pdev, true, false); if (rc) return rc; @@ -4879,6 +4883,7 @@ void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) */ int pci_bridge_secondary_bus_reset(struct pci_dev *dev) { + lock_map_assert_held(&dev->cfg_access_lock); pcibios_reset_secondary_bus(dev); return pci_bridge_wait_for_secondary_bus(dev, "bus reset"); @@ -4927,16 +4932,96 @@ static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe) return pci_reset_hotplug_slot(dev->slot->hotplug, probe); } +static u16 cxl_port_dvsec(struct pci_dev *dev) +{ + return pci_find_dvsec_capability(dev, PCI_VENDOR_ID_CXL, + PCI_DVSEC_CXL_PORT); +} + +static bool cxl_sbr_masked(struct pci_dev *dev) +{ + u16 dvsec, reg; + int rc; + + dvsec = cxl_port_dvsec(dev); + if (!dvsec) + return false; + + rc = pci_read_config_word(dev, dvsec + PCI_DVSEC_CXL_PORT_CTL, ®); + if (rc || PCI_POSSIBLE_ERROR(reg)) + return false; + + /* + * Per CXL spec r3.1, sec 8.1.5.2, when "Unmask SBR" is 0, the SBR + * bit in Bridge Control has no effect. When 1, the Port generates + * hot reset when the SBR bit is set to 1. + */ + if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR) + return false; + + return true; +} + static int pci_reset_bus_function(struct pci_dev *dev, bool probe) { + struct pci_dev *bridge = pci_upstream_bridge(dev); int rc; + /* + * If "dev" is below a CXL port that has SBR control masked, SBR + * won't do anything, so return error. + */ + if (bridge && cxl_sbr_masked(bridge)) { + if (probe) + return 0; + + return -ENOTTY; + } + rc = pci_dev_reset_slot_function(dev, probe); if (rc != -ENOTTY) return rc; return pci_parent_bus_reset(dev, probe); } +static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) +{ + struct pci_dev *bridge; + u16 dvsec, reg, val; + int rc; + + bridge = pci_upstream_bridge(dev); + if (!bridge) + return -ENOTTY; + + dvsec = cxl_port_dvsec(bridge); + if (!dvsec) + return -ENOTTY; + + if (probe) + return 0; + + rc = pci_read_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, ®); + if (rc) + return -ENOTTY; + + if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR) { + val = reg; + } else { + val = reg | PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR; + pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, + val); + } + + rc = pci_reset_bus_function(dev, probe); + + if (reg != val) + pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, + reg); + + return rc; +} + void pci_dev_lock(struct pci_dev *dev) { /* block PM suspend, driver probe, etc. */ @@ -5021,6 +5106,7 @@ static const struct pci_reset_fn_method pci_reset_fn_methods[] = { { pci_af_flr, .name = "af_flr" }, { pci_pm_reset, .name = "pm" }, { pci_reset_bus_function, .name = "bus" }, + { cxl_reset_bus_function, .name = "cxl_bus" }, }; static ssize_t reset_method_show(struct device *dev, @@ -5245,11 +5331,20 @@ void pci_init_reset_methods(struct pci_dev *dev) */ int pci_reset_function(struct pci_dev *dev) { + struct pci_dev *bridge; int rc; if (!pci_reset_supported(dev)) return -ENOTTY; + /* + * If there's no upstream bridge, no locking is needed since there is + * no upstream bridge configuration to hold consistent. + */ + bridge = pci_upstream_bridge(dev); + if (bridge) + pci_dev_lock(bridge); + pci_dev_lock(dev); pci_dev_save_and_disable(dev); @@ -5258,6 +5353,9 @@ int pci_reset_function(struct pci_dev *dev) pci_dev_restore(dev); pci_dev_unlock(dev); + if (bridge) + pci_dev_unlock(bridge); + return rc; } EXPORT_SYMBOL_GPL(pci_reset_function); @@ -6065,8 +6163,9 @@ EXPORT_SYMBOL(pcie_get_width_cap); * and width, multiplying them, and applying encoding overhead. The result * is in Mb/s, i.e., megabits/second of raw bandwidth. */ -u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed, - enum pcie_link_width *width) +static u32 pcie_bandwidth_capable(struct pci_dev *dev, + enum pci_bus_speed *speed, + enum pcie_link_width *width) { *speed = pcie_get_speed_cap(dev); *width = pcie_get_width_cap(dev); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 17fed1846847..fd44565c4756 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -293,8 +293,6 @@ void pci_bus_put(struct pci_bus *bus); const char *pci_speed_string(enum pci_bus_speed speed); enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev); enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev); -u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed, - enum pcie_link_width *width); void __pcie_print_link_status(struct pci_dev *dev, bool verbose); void pcie_report_downtraining(struct pci_dev *dev); void pcie_update_link_speed(struct pci_bus *bus, u16 link_status); diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig index 8999fcebde6a..17919b99fa66 100644 --- a/drivers/pci/pcie/Kconfig +++ b/drivers/pci/pcie/Kconfig @@ -47,7 +47,7 @@ config PCIEAER_INJECT error injection can fake almost all kinds of errors with the help of a user space helper tool aer-inject, which can be gotten from: - https://git.kernel.org/cgit/linux/kernel/git/gong.chen/aer-inject.git/ + https://github.com/intel/aer-inject.git config PCIEAER_CXL bool "PCI Express CXL RAS support" diff --git a/drivers/pci/pcie/aer_inject.c b/drivers/pci/pcie/aer_inject.c index 2dab275d252f..f81b2303bf6a 100644 --- a/drivers/pci/pcie/aer_inject.c +++ b/drivers/pci/pcie/aer_inject.c @@ -6,7 +6,7 @@ * trigger various real hardware errors. Software based error * injection can fake almost all kinds of errors with the help of a * user space helper tool aer-inject, which can be gotten from: - * https://git.kernel.org/cgit/linux/kernel/git/gong.chen/aer-inject.git/ + * https://github.com/intel/aer-inject.git * * Copyright 2009 Intel Corporation. * Huang Ying <ying.huang@intel.com> diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 47761c7ef267..cee2365e54b8 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -8,6 +8,8 @@ */ #include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/build_bug.h> #include <linux/kernel.h> #include <linux/limits.h> #include <linux/math.h> @@ -189,21 +191,18 @@ void pci_restore_aspm_l1ss_state(struct pci_dev *pdev) #endif #define MODULE_PARAM_PREFIX "pcie_aspm." -/* Note: those are not register definitions */ -#define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ -#define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ -#define ASPM_STATE_L1 (4) /* L1 state */ -#define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */ -#define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */ -#define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */ -#define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */ -#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) -#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) -#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ - ASPM_STATE_L1_2_MASK) -#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) -#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ - ASPM_STATE_L1SS) +/* Note: these are not register definitions */ +#define PCIE_LINK_STATE_L0S_UP BIT(0) /* Upstream direction L0s state */ +#define PCIE_LINK_STATE_L0S_DW BIT(1) /* Downstream direction L0s state */ +static_assert(PCIE_LINK_STATE_L0S == (PCIE_LINK_STATE_L0S_UP | PCIE_LINK_STATE_L0S_DW)); + +#define PCIE_LINK_STATE_L1_SS_PCIPM (PCIE_LINK_STATE_L1_1_PCIPM |\ + PCIE_LINK_STATE_L1_2_PCIPM) +#define PCIE_LINK_STATE_L1_2_MASK (PCIE_LINK_STATE_L1_2 |\ + PCIE_LINK_STATE_L1_2_PCIPM) +#define PCIE_LINK_STATE_L1SS (PCIE_LINK_STATE_L1_1 |\ + PCIE_LINK_STATE_L1_1_PCIPM |\ + PCIE_LINK_STATE_L1_2_MASK) struct pcie_link_state { struct pci_dev *pdev; /* Upstream component of the Link */ @@ -275,10 +274,10 @@ static int policy_to_aspm_state(struct pcie_link_state *link) return 0; case POLICY_POWERSAVE: /* Enable ASPM L0s/L1 */ - return (ASPM_STATE_L0S | ASPM_STATE_L1); + return PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1; case POLICY_POWER_SUPERSAVE: /* Enable Everything */ - return ASPM_STATE_ALL; + return PCIE_LINK_STATE_ASPM_ALL; case POLICY_DEFAULT: return link->aspm_default; } @@ -581,14 +580,14 @@ static void pcie_aspm_check_latency(struct pci_dev *endpoint) latency_dw_l1 = calc_l1_latency(lnkcap_dw); /* Check upstream direction L0s latency */ - if ((link->aspm_capable & ASPM_STATE_L0S_UP) && + if ((link->aspm_capable & PCIE_LINK_STATE_L0S_UP) && (latency_up_l0s > acceptable_l0s)) - link->aspm_capable &= ~ASPM_STATE_L0S_UP; + link->aspm_capable &= ~PCIE_LINK_STATE_L0S_UP; /* Check downstream direction L0s latency */ - if ((link->aspm_capable & ASPM_STATE_L0S_DW) && + if ((link->aspm_capable & PCIE_LINK_STATE_L0S_DW) && (latency_dw_l0s > acceptable_l0s)) - link->aspm_capable &= ~ASPM_STATE_L0S_DW; + link->aspm_capable &= ~PCIE_LINK_STATE_L0S_DW; /* * Check L1 latency. * Every switch on the path to root complex need 1 @@ -603,9 +602,9 @@ static void pcie_aspm_check_latency(struct pci_dev *endpoint) * substate latencies (and hence do not do any check). */ latency = max_t(u32, latency_up_l1, latency_dw_l1); - if ((link->aspm_capable & ASPM_STATE_L1) && + if ((link->aspm_capable & PCIE_LINK_STATE_L1) && (latency + l1_switch_latency > acceptable_l1)) - link->aspm_capable &= ~ASPM_STATE_L1; + link->aspm_capable &= ~PCIE_LINK_STATE_L1; l1_switch_latency += NSEC_PER_USEC; link = link->parent; @@ -741,13 +740,13 @@ static void aspm_l1ss_init(struct pcie_link_state *link) child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2; if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) - link->aspm_support |= ASPM_STATE_L1_1; + link->aspm_support |= PCIE_LINK_STATE_L1_1; if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) - link->aspm_support |= ASPM_STATE_L1_2; + link->aspm_support |= PCIE_LINK_STATE_L1_2; if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) - link->aspm_support |= ASPM_STATE_L1_1_PCIPM; + link->aspm_support |= PCIE_LINK_STATE_L1_1_PCIPM; if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) - link->aspm_support |= ASPM_STATE_L1_2_PCIPM; + link->aspm_support |= PCIE_LINK_STATE_L1_2_PCIPM; if (parent_l1ss_cap) pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, @@ -757,15 +756,15 @@ static void aspm_l1ss_init(struct pcie_link_state *link) &child_l1ss_ctl1); if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) - link->aspm_enabled |= ASPM_STATE_L1_1; + link->aspm_enabled |= PCIE_LINK_STATE_L1_1; if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) - link->aspm_enabled |= ASPM_STATE_L1_2; + link->aspm_enabled |= PCIE_LINK_STATE_L1_2; if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) - link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; + link->aspm_enabled |= PCIE_LINK_STATE_L1_1_PCIPM; if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) - link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; + link->aspm_enabled |= PCIE_LINK_STATE_L1_2_PCIPM; - if (link->aspm_support & ASPM_STATE_L1_2_MASK) + if (link->aspm_support & PCIE_LINK_STATE_L1_2_MASK) aspm_calc_l12_info(link, parent_l1ss_cap, child_l1ss_cap); } @@ -778,8 +777,8 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) if (blacklist) { /* Set enabled/disable so that we will disable ASPM later */ - link->aspm_enabled = ASPM_STATE_ALL; - link->aspm_disable = ASPM_STATE_ALL; + link->aspm_enabled = PCIE_LINK_STATE_ASPM_ALL; + link->aspm_disable = PCIE_LINK_STATE_ASPM_ALL; return; } @@ -814,19 +813,19 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) * support L0s. */ if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S) - link->aspm_support |= ASPM_STATE_L0S; + link->aspm_support |= PCIE_LINK_STATE_L0S; if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S) - link->aspm_enabled |= ASPM_STATE_L0S_UP; + link->aspm_enabled |= PCIE_LINK_STATE_L0S_UP; if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S) - link->aspm_enabled |= ASPM_STATE_L0S_DW; + link->aspm_enabled |= PCIE_LINK_STATE_L0S_DW; /* Setup L1 state */ if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1) - link->aspm_support |= ASPM_STATE_L1; + link->aspm_support |= PCIE_LINK_STATE_L1; if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1) - link->aspm_enabled |= ASPM_STATE_L1; + link->aspm_enabled |= PCIE_LINK_STATE_L1; aspm_l1ss_init(link); @@ -876,7 +875,7 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) * If needed, disable L1, and it gets enabled later * in pcie_config_aspm_link(). */ - if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) { + if (enable_req & (PCIE_LINK_STATE_L1_1 | PCIE_LINK_STATE_L1_2)) { pcie_capability_clear_word(child, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_ASPM_L1); pcie_capability_clear_word(parent, PCI_EXP_LNKCTL, @@ -884,13 +883,13 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) } val = 0; - if (state & ASPM_STATE_L1_1) + if (state & PCIE_LINK_STATE_L1_1) val |= PCI_L1SS_CTL1_ASPM_L1_1; - if (state & ASPM_STATE_L1_2) + if (state & PCIE_LINK_STATE_L1_2) val |= PCI_L1SS_CTL1_ASPM_L1_2; - if (state & ASPM_STATE_L1_1_PCIPM) + if (state & PCIE_LINK_STATE_L1_1_PCIPM) val |= PCI_L1SS_CTL1_PCIPM_L1_1; - if (state & ASPM_STATE_L1_2_PCIPM) + if (state & PCIE_LINK_STATE_L1_2_PCIPM) val |= PCI_L1SS_CTL1_PCIPM_L1_2; /* Enable what we need to enable */ @@ -916,29 +915,29 @@ static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) state &= (link->aspm_capable & ~link->aspm_disable); /* Can't enable any substates if L1 is not enabled */ - if (!(state & ASPM_STATE_L1)) - state &= ~ASPM_STATE_L1SS; + if (!(state & PCIE_LINK_STATE_L1)) + state &= ~PCIE_LINK_STATE_L1SS; /* Spec says both ports must be in D0 before enabling PCI PM substates*/ if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) { - state &= ~ASPM_STATE_L1_SS_PCIPM; - state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM); + state &= ~PCIE_LINK_STATE_L1_SS_PCIPM; + state |= (link->aspm_enabled & PCIE_LINK_STATE_L1_SS_PCIPM); } /* Nothing to do if the link is already in the requested state */ if (link->aspm_enabled == state) return; /* Convert ASPM state to upstream/downstream ASPM register state */ - if (state & ASPM_STATE_L0S_UP) + if (state & PCIE_LINK_STATE_L0S_UP) dwstream |= PCI_EXP_LNKCTL_ASPM_L0S; - if (state & ASPM_STATE_L0S_DW) + if (state & PCIE_LINK_STATE_L0S_DW) upstream |= PCI_EXP_LNKCTL_ASPM_L0S; - if (state & ASPM_STATE_L1) { + if (state & PCIE_LINK_STATE_L1) { upstream |= PCI_EXP_LNKCTL_ASPM_L1; dwstream |= PCI_EXP_LNKCTL_ASPM_L1; } - if (link->aspm_capable & ASPM_STATE_L1SS) + if (link->aspm_capable & PCIE_LINK_STATE_L1SS) pcie_config_aspm_l1ss(link, state); /* @@ -947,11 +946,11 @@ static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) * upstream component first and then downstream, and vice * versa for disabling ASPM L1. Spec doesn't mention L0S. */ - if (state & ASPM_STATE_L1) + if (state & PCIE_LINK_STATE_L1) pcie_config_aspm_dev(parent, upstream); list_for_each_entry(child, &linkbus->devices, bus_list) pcie_config_aspm_dev(child, dwstream); - if (!(state & ASPM_STATE_L1)) + if (!(state & PCIE_LINK_STATE_L1)) pcie_config_aspm_dev(parent, upstream); link->aspm_enabled = state; @@ -1324,6 +1323,28 @@ static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev) return bridge->link_state; } +static u8 pci_calc_aspm_disable_mask(int state) +{ + state &= ~PCIE_LINK_STATE_CLKPM; + + /* L1 PM substates require L1 */ + if (state & PCIE_LINK_STATE_L1) + state |= PCIE_LINK_STATE_L1SS; + + return state; +} + +static u8 pci_calc_aspm_enable_mask(int state) +{ + state &= ~PCIE_LINK_STATE_CLKPM; + + /* L1 PM substates require L1 */ + if (state & PCIE_LINK_STATE_L1SS) + state |= PCIE_LINK_STATE_L1; + + return state; +} + static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool locked) { struct pcie_link_state *link = pcie_aspm_get_link(pdev); @@ -1346,19 +1367,7 @@ static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool locked if (!locked) down_read(&pci_bus_sem); mutex_lock(&aspm_lock); - if (state & PCIE_LINK_STATE_L0S) - link->aspm_disable |= ASPM_STATE_L0S; - if (state & PCIE_LINK_STATE_L1) - /* L1 PM substates require L1 */ - link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS; - if (state & PCIE_LINK_STATE_L1_1) - link->aspm_disable |= ASPM_STATE_L1_1; - if (state & PCIE_LINK_STATE_L1_2) - link->aspm_disable |= ASPM_STATE_L1_2; - if (state & PCIE_LINK_STATE_L1_1_PCIPM) - link->aspm_disable |= ASPM_STATE_L1_1_PCIPM; - if (state & PCIE_LINK_STATE_L1_2_PCIPM) - link->aspm_disable |= ASPM_STATE_L1_2_PCIPM; + link->aspm_disable |= pci_calc_aspm_disable_mask(state); pcie_config_aspm_link(link, policy_to_aspm_state(link)); if (state & PCIE_LINK_STATE_CLKPM) @@ -1414,20 +1423,7 @@ static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked) if (!locked) down_read(&pci_bus_sem); mutex_lock(&aspm_lock); - link->aspm_default = 0; - if (state & PCIE_LINK_STATE_L0S) - link->aspm_default |= ASPM_STATE_L0S; - if (state & PCIE_LINK_STATE_L1) - link->aspm_default |= ASPM_STATE_L1; - /* L1 PM substates require L1 */ - if (state & PCIE_LINK_STATE_L1_1) - link->aspm_default |= ASPM_STATE_L1_1 | ASPM_STATE_L1; - if (state & PCIE_LINK_STATE_L1_2) - link->aspm_default |= ASPM_STATE_L1_2 | ASPM_STATE_L1; - if (state & PCIE_LINK_STATE_L1_1_PCIPM) - link->aspm_default |= ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1; - if (state & PCIE_LINK_STATE_L1_2_PCIPM) - link->aspm_default |= ASPM_STATE_L1_2_PCIPM | ASPM_STATE_L1; + link->aspm_default = pci_calc_aspm_enable_mask(state); pcie_config_aspm_link(link, policy_to_aspm_state(link)); link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0; @@ -1563,12 +1559,12 @@ static ssize_t aspm_attr_store_common(struct device *dev, if (state_enable) { link->aspm_disable &= ~state; /* need to enable L1 for substates */ - if (state & ASPM_STATE_L1SS) - link->aspm_disable &= ~ASPM_STATE_L1; + if (state & PCIE_LINK_STATE_L1SS) + link->aspm_disable &= ~PCIE_LINK_STATE_L1; } else { link->aspm_disable |= state; - if (state & ASPM_STATE_L1) - link->aspm_disable |= ASPM_STATE_L1SS; + if (state & PCIE_LINK_STATE_L1) + link->aspm_disable |= PCIE_LINK_STATE_L1SS; } pcie_config_aspm_link(link, policy_to_aspm_state(link)); @@ -1582,12 +1578,12 @@ static ssize_t aspm_attr_store_common(struct device *dev, #define ASPM_ATTR(_f, _s) \ static ssize_t _f##_show(struct device *dev, \ struct device_attribute *attr, char *buf) \ -{ return aspm_attr_show_common(dev, attr, buf, ASPM_STATE_##_s); } \ +{ return aspm_attr_show_common(dev, attr, buf, PCIE_LINK_STATE_##_s); } \ \ static ssize_t _f##_store(struct device *dev, \ struct device_attribute *attr, \ const char *buf, size_t len) \ -{ return aspm_attr_store_common(dev, attr, buf, len, ASPM_STATE_##_s); } +{ return aspm_attr_store_common(dev, attr, buf, len, PCIE_LINK_STATE_##_s); } ASPM_ATTR(l0s_aspm, L0S) ASPM_ATTR(l1_aspm, L1) @@ -1654,12 +1650,12 @@ static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj, struct pci_dev *pdev = to_pci_dev(dev); struct pcie_link_state *link = pcie_aspm_get_link(pdev); static const u8 aspm_state_map[] = { - ASPM_STATE_L0S, - ASPM_STATE_L1, - ASPM_STATE_L1_1, - ASPM_STATE_L1_2, - ASPM_STATE_L1_1_PCIPM, - ASPM_STATE_L1_2_PCIPM, + PCIE_LINK_STATE_L0S, + PCIE_LINK_STATE_L1, + PCIE_LINK_STATE_L1_1, + PCIE_LINK_STATE_L1_2, + PCIE_LINK_STATE_L1_1_PCIPM, + PCIE_LINK_STATE_L1_2_PCIPM, }; if (aspm_disabled || !link) diff --git a/drivers/pci/pcie/edr.c b/drivers/pci/pcie/edr.c index 5f4914d313a1..e86298dbbcff 100644 --- a/drivers/pci/pcie/edr.c +++ b/drivers/pci/pcie/edr.c @@ -32,10 +32,10 @@ static int acpi_enable_dpc(struct pci_dev *pdev) int status = 0; /* - * Behavior when calling unsupported _DSM functions is undefined, - * so check whether EDR_PORT_DPC_ENABLE_DSM is supported. + * Per PCI Firmware r3.3, sec 4.6.12, EDR_PORT_DPC_ENABLE_DSM is + * optional. Return success if it's not implemented. */ - if (!acpi_check_dsm(adev->handle, &pci_acpi_dsm_guid, 5, + if (!acpi_check_dsm(adev->handle, &pci_acpi_dsm_guid, 6, 1ULL << EDR_PORT_DPC_ENABLE_DSM)) return 0; @@ -46,12 +46,7 @@ static int acpi_enable_dpc(struct pci_dev *pdev) argv4.package.count = 1; argv4.package.elements = &req; - /* - * Per Downstream Port Containment Related Enhancements ECN to PCI - * Firmware Specification r3.2, sec 4.6.12, EDR_PORT_DPC_ENABLE_DSM is - * optional. Return success if it's not implemented. - */ - obj = acpi_evaluate_dsm(adev->handle, &pci_acpi_dsm_guid, 5, + obj = acpi_evaluate_dsm(adev->handle, &pci_acpi_dsm_guid, 6, EDR_PORT_DPC_ENABLE_DSM, &argv4); if (!obj) return 0; @@ -85,8 +80,9 @@ static struct pci_dev *acpi_dpc_port_get(struct pci_dev *pdev) u16 port; /* - * Behavior when calling unsupported _DSM functions is undefined, - * so check whether EDR_PORT_DPC_ENABLE_DSM is supported. + * If EDR_PORT_LOCATE_DSM is not implemented under the target of + * EDR, the target is the port that experienced the containment + * event (PCI Firmware r3.3, sec 4.6.13). */ if (!acpi_check_dsm(adev->handle, &pci_acpi_dsm_guid, 5, 1ULL << EDR_PORT_LOCATE_DSM)) @@ -104,6 +100,16 @@ static struct pci_dev *acpi_dpc_port_get(struct pci_dev *pdev) } /* + * Bit 31 represents the success/failure of the operation. If bit + * 31 is set, the operation failed. + */ + if (obj->integer.value & BIT(31)) { + ACPI_FREE(obj); + pci_err(pdev, "Locate Port _DSM failed\n"); + return NULL; + } + + /* * Firmware returns DPC port BDF details in following format: * 15:8 = bus * 7:3 = device diff --git a/drivers/pci/pcie/err.c b/drivers/pci/pcie/err.c index 705893b5f7b0..31090770fffc 100644 --- a/drivers/pci/pcie/err.c +++ b/drivers/pci/pcie/err.c @@ -116,9 +116,7 @@ static int report_mmio_enabled(struct pci_dev *dev, void *data) device_lock(&dev->dev); pdrv = dev->driver; - if (!pdrv || - !pdrv->err_handler || - !pdrv->err_handler->mmio_enabled) + if (!pdrv || !pdrv->err_handler || !pdrv->err_handler->mmio_enabled) goto out; err_handler = pdrv->err_handler; @@ -137,9 +135,7 @@ static int report_slot_reset(struct pci_dev *dev, void *data) device_lock(&dev->dev); pdrv = dev->driver; - if (!pdrv || - !pdrv->err_handler || - !pdrv->err_handler->slot_reset) + if (!pdrv || !pdrv->err_handler || !pdrv->err_handler->slot_reset) goto out; err_handler = pdrv->err_handler; @@ -158,9 +154,7 @@ static int report_resume(struct pci_dev *dev, void *data) device_lock(&dev->dev); pdrv = dev->driver; if (!pci_dev_set_io_state(dev, pci_channel_io_normal) || - !pdrv || - !pdrv->err_handler || - !pdrv->err_handler->resume) + !pdrv || !pdrv->err_handler || !pdrv->err_handler->resume) goto out; err_handler = pdrv->err_handler; diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index 14a4b89a3b83..bb65dfe43409 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -187,15 +187,15 @@ static int pcie_init_service_irqs(struct pci_dev *dev, int *irqs, int mask) * interrupt. */ if ((mask & PCIE_PORT_SERVICE_PME) && pcie_pme_no_msi()) - goto legacy_irq; + goto intx_irq; /* Try to use MSI-X or MSI if supported */ if (pcie_port_enable_irq_vec(dev, irqs, mask) == 0) return 0; -legacy_irq: - /* fall back to legacy IRQ */ - ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_LEGACY); +intx_irq: + /* fall back to INTX IRQ */ + ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_INTX); if (ret < 0) return -ENODEV; diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 1325fbae2f28..8e696e547565 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -95,7 +95,7 @@ static void release_pcibus_dev(struct device *dev) kfree(pci_bus); } -static struct class pcibus_class = { +static const struct class pcibus_class = { .name = "pci_bus", .dev_release = &release_pcibus_dev, .dev_groups = pcibus_groups, @@ -1482,6 +1482,9 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, } out: + /* Clear errors in the Secondary Status Register */ + pci_write_config_word(dev, PCI_SEC_STATUS, 0xffff); + pci_write_config_word(dev, PCI_BRIDGE_CONTROL, bctl); pm_runtime_put(&dev->dev); @@ -2543,6 +2546,9 @@ void pci_device_add(struct pci_dev *dev, struct pci_bus *bus) dev->dev.dma_mask = &dev->dma_mask; dev->dev.dma_parms = &dev->dma_parms; dev->dev.coherent_dma_mask = 0xffffffffull; + lockdep_register_key(&dev->cfg_access_key); + lockdep_init_map(&dev->cfg_access_lock, dev_name(&dev->dev), + &dev->cfg_access_key, 0); dma_set_max_seg_size(&dev->dev, 65536); dma_set_seg_boundary(&dev->dev, 0xffffffff); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index eff7f5df08e2..568410e64ce6 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -6253,3 +6253,23 @@ static void pci_fixup_d3cold_delay_1sec(struct pci_dev *pdev) pdev->d3cold_delay = 1000; } DECLARE_PCI_FIXUP_FINAL(0x5555, 0x0004, pci_fixup_d3cold_delay_1sec); + +#ifdef CONFIG_PCIEAER +static void pci_mask_replay_timer_timeout(struct pci_dev *pdev) +{ + struct pci_dev *parent = pci_upstream_bridge(pdev); + u32 val; + + if (!parent || !parent->aer_cap) + return; + + pci_info(parent, "mask Replay Timer Timeout Correctable Errors due to %s hardware defect", + pci_name(pdev)); + + pci_read_config_dword(parent, parent->aer_cap + PCI_ERR_COR_MASK, &val); + val |= PCI_ERR_COR_REP_TIMER; + pci_write_config_dword(parent, parent->aer_cap + PCI_ERR_COR_MASK, val); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_GLI, 0x9750, pci_mask_replay_timer_timeout); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_GLI, 0x9755, pci_mask_replay_timer_timeout); +#endif diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index 308c9969642e..a1b742b1a735 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -345,7 +345,7 @@ static ssize_t cxl_pmu_event_sysfs_show(struct device *dev, /* For CXL spec defined events */ #define CXL_PMU_EVENT_CXL_ATTR(_name, _gid, _msk) \ - CXL_PMU_EVENT_ATTR(_name, PCI_DVSEC_VENDOR_ID_CXL, _gid, _msk) + CXL_PMU_EVENT_ATTR(_name, PCI_VENDOR_ID_CXL, _gid, _msk) static struct attribute *cxl_pmu_event_attrs[] = { CXL_PMU_EVENT_CXL_ATTR(clock_ticks, CXL_PMU_GID_CLOCK_TICKS, BIT(0)), diff --git a/drivers/phy/freescale/Kconfig b/drivers/phy/freescale/Kconfig index 853958fb2c06..45aaaea14fb4 100644 --- a/drivers/phy/freescale/Kconfig +++ b/drivers/phy/freescale/Kconfig @@ -35,6 +35,12 @@ config PHY_FSL_IMX8M_PCIE Enable this to add support for the PCIE PHY as found on i.MX8M family of SOCs. +config PHY_FSL_SAMSUNG_HDMI_PHY + tristate "Samsung HDMI PHY support" + depends on OF && HAS_IOMEM && COMMON_CLK + help + Enable this to add support for the Samsung HDMI PHY in i.MX8MP. + endif config PHY_FSL_LYNX_28G diff --git a/drivers/phy/freescale/Makefile b/drivers/phy/freescale/Makefile index cedb328bc4d2..c4386bfdb853 100644 --- a/drivers/phy/freescale/Makefile +++ b/drivers/phy/freescale/Makefile @@ -4,3 +4,4 @@ obj-$(CONFIG_PHY_MIXEL_LVDS_PHY) += phy-fsl-imx8qm-lvds-phy.o obj-$(CONFIG_PHY_MIXEL_MIPI_DPHY) += phy-fsl-imx8-mipi-dphy.o obj-$(CONFIG_PHY_FSL_IMX8M_PCIE) += phy-fsl-imx8m-pcie.o obj-$(CONFIG_PHY_FSL_LYNX_28G) += phy-fsl-lynx-28g.o +obj-$(CONFIG_PHY_FSL_SAMSUNG_HDMI_PHY) += phy-fsl-samsung-hdmi.o diff --git a/drivers/phy/freescale/phy-fsl-samsung-hdmi.c b/drivers/phy/freescale/phy-fsl-samsung-hdmi.c new file mode 100644 index 000000000000..9048cdc760c2 --- /dev/null +++ b/drivers/phy/freescale/phy-fsl-samsung-hdmi.c @@ -0,0 +1,718 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2020 NXP + * Copyright 2022 Pengutronix, Lucas Stach <kernel@pengutronix.de> + */ + +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/delay.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> + +#define PHY_REG_00 0x00 +#define PHY_REG_01 0x04 +#define PHY_REG_02 0x08 +#define PHY_REG_08 0x20 +#define PHY_REG_09 0x24 +#define PHY_REG_10 0x28 +#define PHY_REG_11 0x2c + +#define PHY_REG_12 0x30 +#define REG12_CK_DIV_MASK GENMASK(5, 4) + +#define PHY_REG_13 0x34 +#define REG13_TG_CODE_LOW_MASK GENMASK(7, 0) + +#define PHY_REG_14 0x38 +#define REG14_TOL_MASK GENMASK(7, 4) +#define REG14_RP_CODE_MASK GENMASK(3, 1) +#define REG14_TG_CODE_HIGH_MASK GENMASK(0, 0) + +#define PHY_REG_15 0x3c +#define PHY_REG_16 0x40 +#define PHY_REG_17 0x44 +#define PHY_REG_18 0x48 +#define PHY_REG_19 0x4c +#define PHY_REG_20 0x50 + +#define PHY_REG_21 0x54 +#define REG21_SEL_TX_CK_INV BIT(7) +#define REG21_PMS_S_MASK GENMASK(3, 0) + +#define PHY_REG_22 0x58 +#define PHY_REG_23 0x5c +#define PHY_REG_24 0x60 +#define PHY_REG_25 0x64 +#define PHY_REG_26 0x68 +#define PHY_REG_27 0x6c +#define PHY_REG_28 0x70 +#define PHY_REG_29 0x74 +#define PHY_REG_30 0x78 +#define PHY_REG_31 0x7c +#define PHY_REG_32 0x80 + +/* + * REG33 does not match the ref manual. According to Sandor Yu from NXP, + * "There is a doc issue on the i.MX8MP latest RM" + * REG33 is being used per guidance from Sandor + */ + +#define PHY_REG_33 0x84 +#define REG33_MODE_SET_DONE BIT(7) +#define REG33_FIX_DA BIT(1) + +#define PHY_REG_34 0x88 +#define REG34_PHY_READY BIT(7) +#define REG34_PLL_LOCK BIT(6) +#define REG34_PHY_CLK_READY BIT(5) + +#define PHY_REG_35 0x8c +#define PHY_REG_36 0x90 +#define PHY_REG_37 0x94 +#define PHY_REG_38 0x98 +#define PHY_REG_39 0x9c +#define PHY_REG_40 0xa0 +#define PHY_REG_41 0xa4 +#define PHY_REG_42 0xa8 +#define PHY_REG_43 0xac +#define PHY_REG_44 0xb0 +#define PHY_REG_45 0xb4 +#define PHY_REG_46 0xb8 +#define PHY_REG_47 0xbc + +#define PHY_PLL_DIV_REGS_NUM 6 + +struct phy_config { + u32 pixclk; + u8 pll_div_regs[PHY_PLL_DIV_REGS_NUM]; +}; + +static const struct phy_config phy_pll_cfg[] = { + { + .pixclk = 22250000, + .pll_div_regs = { 0x4b, 0xf1, 0x89, 0x88, 0x80, 0x40 }, + }, { + .pixclk = 23750000, + .pll_div_regs = { 0x50, 0xf1, 0x86, 0x85, 0x80, 0x40 }, + }, { + .pixclk = 24000000, + .pll_div_regs = { 0x50, 0xf0, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 24024000, + .pll_div_regs = { 0x50, 0xf1, 0x99, 0x02, 0x80, 0x40 }, + }, { + .pixclk = 25175000, + .pll_div_regs = { 0x54, 0xfc, 0xcc, 0x91, 0x80, 0x40 }, + }, { + .pixclk = 25200000, + .pll_div_regs = { 0x54, 0xf0, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 26750000, + .pll_div_regs = { 0x5a, 0xf2, 0x89, 0x88, 0x80, 0x40 }, + }, { + .pixclk = 27000000, + .pll_div_regs = { 0x5a, 0xf0, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 27027000, + .pll_div_regs = { 0x5a, 0xf2, 0xfd, 0x0c, 0x80, 0x40 }, + }, { + .pixclk = 29500000, + .pll_div_regs = { 0x62, 0xf4, 0x95, 0x08, 0x80, 0x40 }, + }, { + .pixclk = 30750000, + .pll_div_regs = { 0x66, 0xf4, 0x82, 0x01, 0x88, 0x45 }, + }, { + .pixclk = 30888000, + .pll_div_regs = { 0x66, 0xf4, 0x99, 0x18, 0x88, 0x45 }, + }, { + .pixclk = 33750000, + .pll_div_regs = { 0x70, 0xf4, 0x82, 0x01, 0x80, 0x40 }, + }, { + .pixclk = 35000000, + .pll_div_regs = { 0x58, 0xb8, 0x8b, 0x88, 0x80, 0x40 }, + }, { + .pixclk = 36000000, + .pll_div_regs = { 0x5a, 0xb0, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 36036000, + .pll_div_regs = { 0x5a, 0xb2, 0xfd, 0x0c, 0x80, 0x40 }, + }, { + .pixclk = 40000000, + .pll_div_regs = { 0x64, 0xb0, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 43200000, + .pll_div_regs = { 0x5a, 0x90, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 43243200, + .pll_div_regs = { 0x5a, 0x92, 0xfd, 0x0c, 0x80, 0x40 }, + }, { + .pixclk = 44500000, + .pll_div_regs = { 0x5c, 0x92, 0x98, 0x11, 0x84, 0x41 }, + }, { + .pixclk = 47000000, + .pll_div_regs = { 0x62, 0x94, 0x95, 0x82, 0x80, 0x40 }, + }, { + .pixclk = 47500000, + .pll_div_regs = { 0x63, 0x96, 0xa1, 0x82, 0x80, 0x40 }, + }, { + .pixclk = 50349650, + .pll_div_regs = { 0x54, 0x7c, 0xc3, 0x8f, 0x80, 0x40 }, + }, { + .pixclk = 50400000, + .pll_div_regs = { 0x54, 0x70, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 53250000, + .pll_div_regs = { 0x58, 0x72, 0x84, 0x03, 0x82, 0x41 }, + }, { + .pixclk = 53500000, + .pll_div_regs = { 0x5a, 0x72, 0x89, 0x88, 0x80, 0x40 }, + }, { + .pixclk = 54000000, + .pll_div_regs = { 0x5a, 0x70, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 54054000, + .pll_div_regs = { 0x5a, 0x72, 0xfd, 0x0c, 0x80, 0x40 }, + }, { + .pixclk = 59000000, + .pll_div_regs = { 0x62, 0x74, 0x95, 0x08, 0x80, 0x40 }, + }, { + .pixclk = 59340659, + .pll_div_regs = { 0x62, 0x74, 0xdb, 0x52, 0x88, 0x47 }, + }, { + .pixclk = 59400000, + .pll_div_regs = { 0x63, 0x70, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 61500000, + .pll_div_regs = { 0x66, 0x74, 0x82, 0x01, 0x88, 0x45 }, + }, { + .pixclk = 63500000, + .pll_div_regs = { 0x69, 0x74, 0x89, 0x08, 0x80, 0x40 }, + }, { + .pixclk = 67500000, + .pll_div_regs = { 0x54, 0x52, 0x87, 0x03, 0x80, 0x40 }, + }, { + .pixclk = 70000000, + .pll_div_regs = { 0x58, 0x58, 0x8b, 0x88, 0x80, 0x40 }, + }, { + .pixclk = 72000000, + .pll_div_regs = { 0x5a, 0x50, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 72072000, + .pll_div_regs = { 0x5a, 0x52, 0xfd, 0x0c, 0x80, 0x40 }, + }, { + .pixclk = 74176000, + .pll_div_regs = { 0x5d, 0x58, 0xdb, 0xA2, 0x88, 0x41 }, + }, { + .pixclk = 74250000, + .pll_div_regs = { 0x5c, 0x52, 0x90, 0x0d, 0x84, 0x41 }, + }, { + .pixclk = 78500000, + .pll_div_regs = { 0x62, 0x54, 0x87, 0x01, 0x80, 0x40 }, + }, { + .pixclk = 80000000, + .pll_div_regs = { 0x64, 0x50, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 82000000, + .pll_div_regs = { 0x66, 0x54, 0x82, 0x01, 0x88, 0x45 }, + }, { + .pixclk = 82500000, + .pll_div_regs = { 0x67, 0x54, 0x88, 0x01, 0x90, 0x49 }, + }, { + .pixclk = 89000000, + .pll_div_regs = { 0x70, 0x54, 0x84, 0x83, 0x80, 0x40 }, + }, { + .pixclk = 90000000, + .pll_div_regs = { 0x70, 0x54, 0x82, 0x01, 0x80, 0x40 }, + }, { + .pixclk = 94000000, + .pll_div_regs = { 0x4e, 0x32, 0xa7, 0x10, 0x80, 0x40 }, + }, { + .pixclk = 95000000, + .pll_div_regs = { 0x50, 0x31, 0x86, 0x85, 0x80, 0x40 }, + }, { + .pixclk = 98901099, + .pll_div_regs = { 0x52, 0x3a, 0xdb, 0x4c, 0x88, 0x47 }, + }, { + .pixclk = 99000000, + .pll_div_regs = { 0x52, 0x32, 0x82, 0x01, 0x88, 0x47 }, + }, { + .pixclk = 100699300, + .pll_div_regs = { 0x54, 0x3c, 0xc3, 0x8f, 0x80, 0x40 }, + }, { + .pixclk = 100800000, + .pll_div_regs = { 0x54, 0x30, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 102500000, + .pll_div_regs = { 0x55, 0x32, 0x8c, 0x05, 0x90, 0x4b }, + }, { + .pixclk = 104750000, + .pll_div_regs = { 0x57, 0x32, 0x98, 0x07, 0x90, 0x49 }, + }, { + .pixclk = 106500000, + .pll_div_regs = { 0x58, 0x32, 0x84, 0x03, 0x82, 0x41 }, + }, { + .pixclk = 107000000, + .pll_div_regs = { 0x5a, 0x32, 0x89, 0x88, 0x80, 0x40 }, + }, { + .pixclk = 108000000, + .pll_div_regs = { 0x5a, 0x30, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 108108000, + .pll_div_regs = { 0x5a, 0x32, 0xfd, 0x0c, 0x80, 0x40 }, + }, { + .pixclk = 118000000, + .pll_div_regs = { 0x62, 0x34, 0x95, 0x08, 0x80, 0x40 }, + }, { + .pixclk = 118800000, + .pll_div_regs = { 0x63, 0x30, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 123000000, + .pll_div_regs = { 0x66, 0x34, 0x82, 0x01, 0x88, 0x45 }, + }, { + .pixclk = 127000000, + .pll_div_regs = { 0x69, 0x34, 0x89, 0x08, 0x80, 0x40 }, + }, { + .pixclk = 135000000, + .pll_div_regs = { 0x70, 0x34, 0x82, 0x01, 0x80, 0x40 }, + }, { + .pixclk = 135580000, + .pll_div_regs = { 0x71, 0x39, 0xe9, 0x82, 0x9c, 0x5b }, + }, { + .pixclk = 137520000, + .pll_div_regs = { 0x72, 0x38, 0x99, 0x10, 0x85, 0x41 }, + }, { + .pixclk = 138750000, + .pll_div_regs = { 0x73, 0x35, 0x88, 0x05, 0x90, 0x4d }, + }, { + .pixclk = 140000000, + .pll_div_regs = { 0x75, 0x36, 0xa7, 0x90, 0x80, 0x40 }, + }, { + .pixclk = 144000000, + .pll_div_regs = { 0x78, 0x30, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 148352000, + .pll_div_regs = { 0x7b, 0x35, 0xdb, 0x39, 0x90, 0x45 }, + }, { + .pixclk = 148500000, + .pll_div_regs = { 0x7b, 0x35, 0x84, 0x03, 0x90, 0x45 }, + }, { + .pixclk = 154000000, + .pll_div_regs = { 0x40, 0x18, 0x83, 0x01, 0x00, 0x40 }, + }, { + .pixclk = 157000000, + .pll_div_regs = { 0x41, 0x11, 0xa7, 0x14, 0x80, 0x40 }, + }, { + .pixclk = 160000000, + .pll_div_regs = { 0x42, 0x12, 0xa1, 0x20, 0x80, 0x40 }, + }, { + .pixclk = 162000000, + .pll_div_regs = { 0x43, 0x18, 0x8b, 0x08, 0x96, 0x55 }, + }, { + .pixclk = 164000000, + .pll_div_regs = { 0x45, 0x11, 0x83, 0x82, 0x90, 0x4b }, + }, { + .pixclk = 165000000, + .pll_div_regs = { 0x45, 0x11, 0x84, 0x81, 0x90, 0x4b }, + }, { + .pixclk = 180000000, + .pll_div_regs = { 0x4b, 0x10, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 185625000, + .pll_div_regs = { 0x4e, 0x12, 0x9a, 0x95, 0x80, 0x40 }, + }, { + .pixclk = 188000000, + .pll_div_regs = { 0x4e, 0x12, 0xa7, 0x10, 0x80, 0x40 }, + }, { + .pixclk = 198000000, + .pll_div_regs = { 0x52, 0x12, 0x82, 0x01, 0x88, 0x47 }, + }, { + .pixclk = 205000000, + .pll_div_regs = { 0x55, 0x12, 0x8c, 0x05, 0x90, 0x4b }, + }, { + .pixclk = 209500000, + .pll_div_regs = { 0x57, 0x12, 0x98, 0x07, 0x90, 0x49 }, + }, { + .pixclk = 213000000, + .pll_div_regs = { 0x58, 0x12, 0x84, 0x03, 0x82, 0x41 }, + }, { + .pixclk = 216000000, + .pll_div_regs = { 0x5a, 0x10, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 216216000, + .pll_div_regs = { 0x5a, 0x12, 0xfd, 0x0c, 0x80, 0x40 }, + }, { + .pixclk = 237600000, + .pll_div_regs = { 0x63, 0x10, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 254000000, + .pll_div_regs = { 0x69, 0x14, 0x89, 0x08, 0x80, 0x40 }, + }, { + .pixclk = 277500000, + .pll_div_regs = { 0x73, 0x15, 0x88, 0x05, 0x90, 0x4d }, + }, { + .pixclk = 288000000, + .pll_div_regs = { 0x78, 0x10, 0x00, 0x00, 0x80, 0x00 }, + }, { + .pixclk = 297000000, + .pll_div_regs = { 0x7b, 0x15, 0x84, 0x03, 0x90, 0x45 }, + }, +}; + +struct reg_settings { + u8 reg; + u8 val; +}; + +static const struct reg_settings common_phy_cfg[] = { + { PHY_REG_00, 0x00 }, { PHY_REG_01, 0xd1 }, + { PHY_REG_08, 0x4f }, { PHY_REG_09, 0x30 }, + { PHY_REG_10, 0x33 }, { PHY_REG_11, 0x65 }, + /* REG12 pixclk specific */ + /* REG13 pixclk specific */ + /* REG14 pixclk specific */ + { PHY_REG_15, 0x80 }, { PHY_REG_16, 0x6c }, + { PHY_REG_17, 0xf2 }, { PHY_REG_18, 0x67 }, + { PHY_REG_19, 0x00 }, { PHY_REG_20, 0x10 }, + /* REG21 pixclk specific */ + { PHY_REG_22, 0x30 }, { PHY_REG_23, 0x32 }, + { PHY_REG_24, 0x60 }, { PHY_REG_25, 0x8f }, + { PHY_REG_26, 0x00 }, { PHY_REG_27, 0x00 }, + { PHY_REG_28, 0x08 }, { PHY_REG_29, 0x00 }, + { PHY_REG_30, 0x00 }, { PHY_REG_31, 0x00 }, + { PHY_REG_32, 0x00 }, { PHY_REG_33, 0x80 }, + { PHY_REG_34, 0x00 }, { PHY_REG_35, 0x00 }, + { PHY_REG_36, 0x00 }, { PHY_REG_37, 0x00 }, + { PHY_REG_38, 0x00 }, { PHY_REG_39, 0x00 }, + { PHY_REG_40, 0x00 }, { PHY_REG_41, 0xe0 }, + { PHY_REG_42, 0x83 }, { PHY_REG_43, 0x0f }, + { PHY_REG_44, 0x3E }, { PHY_REG_45, 0xf8 }, + { PHY_REG_46, 0x00 }, { PHY_REG_47, 0x00 } +}; + +struct fsl_samsung_hdmi_phy { + struct device *dev; + void __iomem *regs; + struct clk *apbclk; + struct clk *refclk; + + /* clk provider */ + struct clk_hw hw; + const struct phy_config *cur_cfg; +}; + +static inline struct fsl_samsung_hdmi_phy * +to_fsl_samsung_hdmi_phy(struct clk_hw *hw) +{ + return container_of(hw, struct fsl_samsung_hdmi_phy, hw); +} + +static void +fsl_samsung_hdmi_phy_configure_pixclk(struct fsl_samsung_hdmi_phy *phy, + const struct phy_config *cfg) +{ + u8 div = 0x1; + + switch (cfg->pixclk) { + case 22250000 ... 33750000: + div = 0xf; + break; + case 35000000 ... 40000000: + div = 0xb; + break; + case 43200000 ... 47500000: + div = 0x9; + break; + case 50349650 ... 63500000: + div = 0x7; + break; + case 67500000 ... 90000000: + div = 0x5; + break; + case 94000000 ... 148500000: + div = 0x3; + break; + case 154000000 ... 297000000: + div = 0x1; + break; + } + + writeb(REG21_SEL_TX_CK_INV | FIELD_PREP(REG21_PMS_S_MASK, div), + phy->regs + PHY_REG_21); +} + +static void +fsl_samsung_hdmi_phy_configure_pll_lock_det(struct fsl_samsung_hdmi_phy *phy, + const struct phy_config *cfg) +{ + u32 pclk = cfg->pixclk; + u32 fld_tg_code; + u32 pclk_khz; + u8 div = 1; + + switch (cfg->pixclk) { + case 22250000 ... 47500000: + div = 1; + break; + case 50349650 ... 99000000: + div = 2; + break; + case 100699300 ... 198000000: + div = 4; + break; + case 205000000 ... 297000000: + div = 8; + break; + } + + writeb(FIELD_PREP(REG12_CK_DIV_MASK, ilog2(div)), phy->regs + PHY_REG_12); + + /* + * Calculation for the frequency lock detector target code (fld_tg_code) + * is based on reference manual register description of PHY_REG13 + * (13.10.3.1.14.2): + * 1st) Calculate int_pllclk which is determinded by FLD_CK_DIV + * 2nd) Increase resolution to avoid rounding issues + * 3th) Do the div (256 / Freq. of int_pllclk) * 24 + * 4th) Reduce the resolution and always round up since the NXP + * settings rounding up always too. TODO: Check if that is + * correct. + */ + pclk /= div; + pclk_khz = pclk / 1000; + fld_tg_code = 256 * 1000 * 1000 / pclk_khz * 24; + fld_tg_code = DIV_ROUND_UP(fld_tg_code, 1000); + + /* FLD_TOL and FLD_RP_CODE taken from downstream driver */ + writeb(FIELD_PREP(REG13_TG_CODE_LOW_MASK, fld_tg_code), + phy->regs + PHY_REG_13); + writeb(FIELD_PREP(REG14_TOL_MASK, 2) | + FIELD_PREP(REG14_RP_CODE_MASK, 2) | + FIELD_PREP(REG14_TG_CODE_HIGH_MASK, fld_tg_code >> 8), + phy->regs + PHY_REG_14); +} + +static int fsl_samsung_hdmi_phy_configure(struct fsl_samsung_hdmi_phy *phy, + const struct phy_config *cfg) +{ + int i, ret; + u8 val; + + /* HDMI PHY init */ + writeb(REG33_FIX_DA, phy->regs + PHY_REG_33); + + /* common PHY registers */ + for (i = 0; i < ARRAY_SIZE(common_phy_cfg); i++) + writeb(common_phy_cfg[i].val, phy->regs + common_phy_cfg[i].reg); + + /* set individual PLL registers PHY_REG2 ... PHY_REG7 */ + for (i = 0; i < PHY_PLL_DIV_REGS_NUM; i++) + writeb(cfg->pll_div_regs[i], phy->regs + PHY_REG_02 + i * 4); + + fsl_samsung_hdmi_phy_configure_pixclk(phy, cfg); + fsl_samsung_hdmi_phy_configure_pll_lock_det(phy, cfg); + + writeb(REG33_FIX_DA | REG33_MODE_SET_DONE, phy->regs + PHY_REG_33); + + ret = readb_poll_timeout(phy->regs + PHY_REG_34, val, + val & REG34_PLL_LOCK, 50, 20000); + if (ret) + dev_err(phy->dev, "PLL failed to lock\n"); + + return ret; +} + +static unsigned long phy_clk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct fsl_samsung_hdmi_phy *phy = to_fsl_samsung_hdmi_phy(hw); + + if (!phy->cur_cfg) + return 74250000; + + return phy->cur_cfg->pixclk; +} + +static long phy_clk_round_rate(struct clk_hw *hw, + unsigned long rate, unsigned long *parent_rate) +{ + int i; + + for (i = ARRAY_SIZE(phy_pll_cfg) - 1; i >= 0; i--) + if (phy_pll_cfg[i].pixclk <= rate) + return phy_pll_cfg[i].pixclk; + + return -EINVAL; +} + +static int phy_clk_set_rate(struct clk_hw *hw, + unsigned long rate, unsigned long parent_rate) +{ + struct fsl_samsung_hdmi_phy *phy = to_fsl_samsung_hdmi_phy(hw); + int i; + + for (i = ARRAY_SIZE(phy_pll_cfg) - 1; i >= 0; i--) + if (phy_pll_cfg[i].pixclk <= rate) + break; + + if (i < 0) + return -EINVAL; + + phy->cur_cfg = &phy_pll_cfg[i]; + + return fsl_samsung_hdmi_phy_configure(phy, phy->cur_cfg); +} + +static const struct clk_ops phy_clk_ops = { + .recalc_rate = phy_clk_recalc_rate, + .round_rate = phy_clk_round_rate, + .set_rate = phy_clk_set_rate, +}; + +static int phy_clk_register(struct fsl_samsung_hdmi_phy *phy) +{ + struct device *dev = phy->dev; + struct device_node *np = dev->of_node; + struct clk_init_data init; + const char *parent_name; + struct clk *phyclk; + int ret; + + parent_name = __clk_get_name(phy->refclk); + + init.parent_names = &parent_name; + init.num_parents = 1; + init.flags = 0; + init.name = "hdmi_pclk"; + init.ops = &phy_clk_ops; + + phy->hw.init = &init; + + phyclk = devm_clk_register(dev, &phy->hw); + if (IS_ERR(phyclk)) + return dev_err_probe(dev, PTR_ERR(phyclk), + "failed to register clock\n"); + + ret = of_clk_add_hw_provider(np, of_clk_hw_simple_get, phyclk); + if (ret) + return dev_err_probe(dev, ret, + "failed to register clock provider\n"); + + return 0; +} + +static int fsl_samsung_hdmi_phy_probe(struct platform_device *pdev) +{ + struct fsl_samsung_hdmi_phy *phy; + int ret; + + phy = devm_kzalloc(&pdev->dev, sizeof(*phy), GFP_KERNEL); + if (!phy) + return -ENOMEM; + + platform_set_drvdata(pdev, phy); + phy->dev = &pdev->dev; + + phy->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(phy->regs)) + return PTR_ERR(phy->regs); + + phy->apbclk = devm_clk_get(phy->dev, "apb"); + if (IS_ERR(phy->apbclk)) + return dev_err_probe(phy->dev, PTR_ERR(phy->apbclk), + "failed to get apb clk\n"); + + phy->refclk = devm_clk_get(phy->dev, "ref"); + if (IS_ERR(phy->refclk)) + return dev_err_probe(phy->dev, PTR_ERR(phy->refclk), + "failed to get ref clk\n"); + + ret = clk_prepare_enable(phy->apbclk); + if (ret) { + dev_err(phy->dev, "failed to enable apbclk\n"); + return ret; + } + + pm_runtime_get_noresume(phy->dev); + pm_runtime_set_active(phy->dev); + pm_runtime_enable(phy->dev); + + ret = phy_clk_register(phy); + if (ret) { + dev_err(&pdev->dev, "register clk failed\n"); + goto register_clk_failed; + } + + pm_runtime_put(phy->dev); + + return 0; + +register_clk_failed: + clk_disable_unprepare(phy->apbclk); + + return ret; +} + +static void fsl_samsung_hdmi_phy_remove(struct platform_device *pdev) +{ + of_clk_del_provider(pdev->dev.of_node); +} + +static int __maybe_unused fsl_samsung_hdmi_phy_suspend(struct device *dev) +{ + struct fsl_samsung_hdmi_phy *phy = dev_get_drvdata(dev); + + clk_disable_unprepare(phy->apbclk); + + return 0; +} + +static int __maybe_unused fsl_samsung_hdmi_phy_resume(struct device *dev) +{ + struct fsl_samsung_hdmi_phy *phy = dev_get_drvdata(dev); + int ret = 0; + + ret = clk_prepare_enable(phy->apbclk); + if (ret) { + dev_err(phy->dev, "failed to enable apbclk\n"); + return ret; + } + + if (phy->cur_cfg) + ret = fsl_samsung_hdmi_phy_configure(phy, phy->cur_cfg); + + return ret; + +} + +static DEFINE_RUNTIME_DEV_PM_OPS(fsl_samsung_hdmi_phy_pm_ops, + fsl_samsung_hdmi_phy_suspend, + fsl_samsung_hdmi_phy_resume, NULL); + +static const struct of_device_id fsl_samsung_hdmi_phy_of_match[] = { + { + .compatible = "fsl,imx8mp-hdmi-phy", + }, { + /* sentinel */ + } +}; +MODULE_DEVICE_TABLE(of, fsl_samsung_hdmi_phy_of_match); + +static struct platform_driver fsl_samsung_hdmi_phy_driver = { + .probe = fsl_samsung_hdmi_phy_probe, + .remove_new = fsl_samsung_hdmi_phy_remove, + .driver = { + .name = "fsl-samsung-hdmi-phy", + .of_match_table = fsl_samsung_hdmi_phy_of_match, + .pm = pm_ptr(&fsl_samsung_hdmi_phy_pm_ops), + }, +}; +module_platform_driver(fsl_samsung_hdmi_phy_driver); + +MODULE_AUTHOR("Sandor Yu <Sandor.yu@nxp.com>"); +MODULE_DESCRIPTION("SAMSUNG HDMI 2.0 Transmitter PHY Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/mediatek/Kconfig b/drivers/phy/mediatek/Kconfig index 3849b7c87d28..60e00057e8bc 100644 --- a/drivers/phy/mediatek/Kconfig +++ b/drivers/phy/mediatek/Kconfig @@ -13,6 +13,17 @@ config PHY_MTK_PCIE callback for PCIe GEN3 port, it supports software efuse initialization. +config PHY_MTK_XFI_TPHY + tristate "MediaTek 10GE SerDes XFI T-PHY driver" + depends on ARCH_MEDIATEK || COMPILE_TEST + depends on OF + select GENERIC_PHY + help + Say 'Y' here to add support for MediaTek XFI T-PHY driver. + The driver provides access to the Ethernet SerDes T-PHY supporting + 1GE and 2.5GE modes via the LynxI PCS, and 5GE and 10GE modes + via the USXGMII PCS found in MediaTek SoCs with 10G Ethernet. + config PHY_MTK_TPHY tristate "MediaTek T-PHY Driver" depends on ARCH_MEDIATEK || COMPILE_TEST diff --git a/drivers/phy/mediatek/Makefile b/drivers/phy/mediatek/Makefile index f6e24a47e081..1b8088df71e8 100644 --- a/drivers/phy/mediatek/Makefile +++ b/drivers/phy/mediatek/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_PHY_MTK_PCIE) += phy-mtk-pcie.o obj-$(CONFIG_PHY_MTK_TPHY) += phy-mtk-tphy.o obj-$(CONFIG_PHY_MTK_UFS) += phy-mtk-ufs.o obj-$(CONFIG_PHY_MTK_XSPHY) += phy-mtk-xsphy.o +obj-$(CONFIG_PHY_MTK_XFI_TPHY) += phy-mtk-xfi-tphy.o phy-mtk-hdmi-drv-y := phy-mtk-hdmi.o phy-mtk-hdmi-drv-y += phy-mtk-hdmi-mt2701.o diff --git a/drivers/phy/mediatek/phy-mtk-xfi-tphy.c b/drivers/phy/mediatek/phy-mtk-xfi-tphy.c new file mode 100644 index 000000000000..1a0b7484f525 --- /dev/null +++ b/drivers/phy/mediatek/phy-mtk-xfi-tphy.c @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * MediaTek 10GE SerDes XFI T-PHY driver + * + * Copyright (c) 2024 Daniel Golle <daniel@makrotopia.org> + * Bc-bocun Chen <bc-bocun.chen@mediatek.com> + * based on mtk_usxgmii.c and mtk_sgmii.c found in MediaTek's SDK (GPL-2.0) + * Copyright (c) 2022 MediaTek Inc. + * Author: Henry Yen <henry.yen@mediatek.com> + */ + +#include <linux/module.h> +#include <linux/device.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/io.h> +#include <linux/clk.h> +#include <linux/reset.h> +#include <linux/phy.h> +#include <linux/phy/phy.h> + +#include "phy-mtk-io.h" + +#define MTK_XFI_TPHY_NUM_CLOCKS 2 + +#define REG_DIG_GLB_70 0x0070 +#define XTP_PCS_RX_EQ_IN_PROGRESS(x) FIELD_PREP(GENMASK(25, 24), (x)) +#define XTP_PCS_MODE_MASK GENMASK(17, 16) +#define XTP_PCS_MODE(x) FIELD_PREP(GENMASK(17, 16), (x)) +#define XTP_PCS_RST_B BIT(15) +#define XTP_FRC_PCS_RST_B BIT(14) +#define XTP_PCS_PWD_SYNC_MASK GENMASK(13, 12) +#define XTP_PCS_PWD_SYNC(x) FIELD_PREP(XTP_PCS_PWD_SYNC_MASK, (x)) +#define XTP_PCS_PWD_ASYNC_MASK GENMASK(11, 10) +#define XTP_PCS_PWD_ASYNC(x) FIELD_PREP(XTP_PCS_PWD_ASYNC_MASK, (x)) +#define XTP_FRC_PCS_PWD_ASYNC BIT(8) +#define XTP_PCS_UPDT BIT(4) +#define XTP_PCS_IN_FR_RG BIT(0) + +#define REG_DIG_GLB_F4 0x00f4 +#define XFI_DPHY_PCS_SEL BIT(0) +#define XFI_DPHY_PCS_SEL_SGMII FIELD_PREP(XFI_DPHY_PCS_SEL, 1) +#define XFI_DPHY_PCS_SEL_USXGMII FIELD_PREP(XFI_DPHY_PCS_SEL, 0) +#define XFI_DPHY_AD_SGDT_FRC_EN BIT(5) + +#define REG_DIG_LN_TRX_40 0x3040 +#define XTP_LN_FRC_TX_DATA_EN BIT(29) +#define XTP_LN_TX_DATA_EN BIT(28) + +#define REG_DIG_LN_TRX_B0 0x30b0 +#define XTP_LN_FRC_TX_MACCK_EN BIT(5) +#define XTP_LN_TX_MACCK_EN BIT(4) + +#define REG_ANA_GLB_D0 0x90d0 +#define XTP_GLB_USXGMII_SEL_MASK GENMASK(3, 1) +#define XTP_GLB_USXGMII_SEL(x) FIELD_PREP(GENMASK(3, 1), (x)) +#define XTP_GLB_USXGMII_EN BIT(0) + +/** + * struct mtk_xfi_tphy - run-time data of the XFI phy instance + * @base: IO memory area to access phy registers. + * @dev: Kernel device used to output prefixed debug info. + * @reset: Reset control corresponding to the phy instance. + * @clocks: All clocks required for the phy to operate. + * @da_war: Enables work-around for 10GBase-R mode. + */ +struct mtk_xfi_tphy { + void __iomem *base; + struct device *dev; + struct reset_control *reset; + struct clk_bulk_data clocks[MTK_XFI_TPHY_NUM_CLOCKS]; + bool da_war; +}; + +/** + * mtk_xfi_tphy_setup() - Setup phy for specified interface mode. + * @xfi_tphy: XFI phy instance. + * @interface: Ethernet interface mode + * + * The setup function is the condensed result of combining the 5 functions which + * setup the phy in MediaTek's GPL licensed public SDK sources. They can be found + * in mtk_sgmii.c[1] as well as mtk_usxgmii.c[2]. + * + * Many magic values have been replaced by register and bit definitions, however, + * that has not been possible in all cases. While the vendor driver uses a + * sequence of 32-bit writes, here we try to only modify the actually required + * bits. + * + * [1]: https://git01.mediatek.com/plugins/gitiles/openwrt/feeds/mtk-openwrt-feeds/+/b72d6cba92bf9e29fb035c03052fa1e86664a25b/21.02/files/target/linux/mediatek/files-5.4/drivers/net/ethernet/mediatek/mtk_sgmii.c + * + * [2]: https://git01.mediatek.com/plugins/gitiles/openwrt/feeds/mtk-openwrt-feeds/+/dec96a1d9b82cdcda4a56453fd0b453d4cab4b85/21.02/files/target/linux/mediatek/files-5.4/drivers/net/ethernet/mediatek/mtk_eth_soc.c + */ +static void mtk_xfi_tphy_setup(struct mtk_xfi_tphy *xfi_tphy, + phy_interface_t interface) +{ + bool is_1g, is_2p5g, is_5g, is_10g, da_war, use_lynxi_pcs; + + /* shorthands for specific clock speeds depending on interface mode */ + is_1g = interface == PHY_INTERFACE_MODE_1000BASEX || + interface == PHY_INTERFACE_MODE_SGMII; + is_2p5g = interface == PHY_INTERFACE_MODE_2500BASEX; + is_5g = interface == PHY_INTERFACE_MODE_5GBASER; + is_10g = interface == PHY_INTERFACE_MODE_10GBASER || + interface == PHY_INTERFACE_MODE_USXGMII; + + /* Is overriding 10GBase-R tuning value required? */ + da_war = xfi_tphy->da_war && (interface == PHY_INTERFACE_MODE_10GBASER); + + /* configure input mux to either + * - USXGMII PCS (64b/66b coding) for 5G/10G + * - LynxI PCS (8b/10b coding) for 1G/2.5G + */ + use_lynxi_pcs = is_1g || is_2p5g; + + dev_dbg(xfi_tphy->dev, "setting up for mode %s\n", phy_modes(interface)); + + /* Setup PLL setting */ + mtk_phy_update_bits(xfi_tphy->base + 0x9024, 0x100000, is_10g ? 0x0 : 0x100000); + mtk_phy_update_bits(xfi_tphy->base + 0x2020, 0x202000, is_5g ? 0x202000 : 0x0); + mtk_phy_update_bits(xfi_tphy->base + 0x2030, 0x500, is_1g ? 0x0 : 0x500); + mtk_phy_update_bits(xfi_tphy->base + 0x2034, 0xa00, is_1g ? 0x0 : 0xa00); + mtk_phy_update_bits(xfi_tphy->base + 0x2040, 0x340000, is_1g ? 0x200000 : 0x140000); + + /* Setup RXFE BW setting */ + mtk_phy_update_bits(xfi_tphy->base + 0x50f0, 0xc10, is_1g ? 0x410 : is_5g ? 0x800 : 0x400); + mtk_phy_update_bits(xfi_tphy->base + 0x50e0, 0x4000, is_5g ? 0x0 : 0x4000); + + /* Setup RX CDR setting */ + mtk_phy_update_bits(xfi_tphy->base + 0x506c, 0x30000, is_5g ? 0x0 : 0x30000); + mtk_phy_update_bits(xfi_tphy->base + 0x5070, 0x670000, is_5g ? 0x620000 : 0x50000); + mtk_phy_update_bits(xfi_tphy->base + 0x5074, 0x180000, is_5g ? 0x180000 : 0x0); + mtk_phy_update_bits(xfi_tphy->base + 0x5078, 0xf000400, is_5g ? 0x8000000 : + 0x7000400); + mtk_phy_update_bits(xfi_tphy->base + 0x507c, 0x5000500, is_5g ? 0x4000400 : + 0x1000100); + mtk_phy_update_bits(xfi_tphy->base + 0x5080, 0x1410, is_1g ? 0x400 : is_5g ? 0x1010 : 0x0); + mtk_phy_update_bits(xfi_tphy->base + 0x5084, 0x30300, is_1g ? 0x30300 : + is_5g ? 0x30100 : + 0x100); + mtk_phy_update_bits(xfi_tphy->base + 0x5088, 0x60200, is_1g ? 0x20200 : + is_5g ? 0x40000 : + 0x20000); + + /* Setting RXFE adaptation range setting */ + mtk_phy_update_bits(xfi_tphy->base + 0x50e4, 0xc0000, is_5g ? 0x0 : 0xc0000); + mtk_phy_update_bits(xfi_tphy->base + 0x50e8, 0x40000, is_5g ? 0x0 : 0x40000); + mtk_phy_update_bits(xfi_tphy->base + 0x50ec, 0xa00, is_1g ? 0x200 : 0x800); + mtk_phy_update_bits(xfi_tphy->base + 0x50a8, 0xee0000, is_5g ? 0x800000 : + 0x6e0000); + mtk_phy_update_bits(xfi_tphy->base + 0x6004, 0x190000, is_5g ? 0x0 : 0x190000); + + if (is_10g) + writel(0x01423342, xfi_tphy->base + 0x00f8); + else if (is_5g) + writel(0x00a132a1, xfi_tphy->base + 0x00f8); + else if (is_2p5g) + writel(0x009c329c, xfi_tphy->base + 0x00f8); + else + writel(0x00fa32fa, xfi_tphy->base + 0x00f8); + + /* Force SGDT_OUT off and select PCS */ + mtk_phy_update_bits(xfi_tphy->base + REG_DIG_GLB_F4, + XFI_DPHY_AD_SGDT_FRC_EN | XFI_DPHY_PCS_SEL, + XFI_DPHY_AD_SGDT_FRC_EN | + (use_lynxi_pcs ? XFI_DPHY_PCS_SEL_SGMII : + XFI_DPHY_PCS_SEL_USXGMII)); + + /* Force GLB_CKDET_OUT */ + mtk_phy_set_bits(xfi_tphy->base + 0x0030, 0xc00); + + /* Force AEQ on */ + writel(XTP_PCS_RX_EQ_IN_PROGRESS(2) | XTP_PCS_PWD_SYNC(2) | XTP_PCS_PWD_ASYNC(2), + xfi_tphy->base + REG_DIG_GLB_70); + + usleep_range(1, 5); + writel(XTP_LN_FRC_TX_DATA_EN, xfi_tphy->base + REG_DIG_LN_TRX_40); + + /* Setup TX DA default value */ + mtk_phy_update_bits(xfi_tphy->base + 0x30b0, 0x30, 0x20); + writel(0x00008a01, xfi_tphy->base + 0x3028); + writel(0x0000a884, xfi_tphy->base + 0x302c); + writel(0x00083002, xfi_tphy->base + 0x3024); + + /* Setup RG default value */ + if (use_lynxi_pcs) { + writel(0x00011110, xfi_tphy->base + 0x3010); + writel(0x40704000, xfi_tphy->base + 0x3048); + } else { + writel(0x00022220, xfi_tphy->base + 0x3010); + writel(0x0f020a01, xfi_tphy->base + 0x5064); + writel(0x06100600, xfi_tphy->base + 0x50b4); + if (interface == PHY_INTERFACE_MODE_USXGMII) + writel(0x40704000, xfi_tphy->base + 0x3048); + else + writel(0x47684100, xfi_tphy->base + 0x3048); + } + + if (is_1g) + writel(0x0000c000, xfi_tphy->base + 0x3064); + + /* Setup RX EQ initial value */ + mtk_phy_update_bits(xfi_tphy->base + 0x3050, 0xa8000000, + (interface != PHY_INTERFACE_MODE_10GBASER) ? 0xa8000000 : 0x0); + mtk_phy_update_bits(xfi_tphy->base + 0x3054, 0xaa, + (interface != PHY_INTERFACE_MODE_10GBASER) ? 0xaa : 0x0); + + if (!use_lynxi_pcs) + writel(0x00000f00, xfi_tphy->base + 0x306c); + else if (is_2p5g) + writel(0x22000f00, xfi_tphy->base + 0x306c); + else + writel(0x20200f00, xfi_tphy->base + 0x306c); + + mtk_phy_update_bits(xfi_tphy->base + 0xa008, 0x10000, da_war ? 0x10000 : 0x0); + + mtk_phy_update_bits(xfi_tphy->base + 0xa060, 0x50000, use_lynxi_pcs ? 0x50000 : 0x40000); + + /* Setup PHYA speed */ + mtk_phy_update_bits(xfi_tphy->base + REG_ANA_GLB_D0, + XTP_GLB_USXGMII_SEL_MASK | XTP_GLB_USXGMII_EN, + is_10g ? XTP_GLB_USXGMII_SEL(0) : + is_5g ? XTP_GLB_USXGMII_SEL(1) : + is_2p5g ? XTP_GLB_USXGMII_SEL(2) : + XTP_GLB_USXGMII_SEL(3)); + mtk_phy_set_bits(xfi_tphy->base + REG_ANA_GLB_D0, XTP_GLB_USXGMII_EN); + + /* Release reset */ + mtk_phy_set_bits(xfi_tphy->base + REG_DIG_GLB_70, + XTP_PCS_RST_B | XTP_FRC_PCS_RST_B); + usleep_range(150, 500); + + /* Switch to P0 */ + mtk_phy_update_bits(xfi_tphy->base + REG_DIG_GLB_70, + XTP_PCS_IN_FR_RG | + XTP_FRC_PCS_PWD_ASYNC | + XTP_PCS_PWD_ASYNC_MASK | + XTP_PCS_PWD_SYNC_MASK | + XTP_PCS_UPDT, + XTP_PCS_IN_FR_RG | + XTP_FRC_PCS_PWD_ASYNC | + XTP_PCS_UPDT); + usleep_range(1, 5); + + mtk_phy_clear_bits(xfi_tphy->base + REG_DIG_GLB_70, XTP_PCS_UPDT); + usleep_range(15, 50); + + if (use_lynxi_pcs) { + /* Switch to Gen2 */ + mtk_phy_update_bits(xfi_tphy->base + REG_DIG_GLB_70, + XTP_PCS_MODE_MASK | XTP_PCS_UPDT, + XTP_PCS_MODE(1) | XTP_PCS_UPDT); + } else { + /* Switch to Gen3 */ + mtk_phy_update_bits(xfi_tphy->base + REG_DIG_GLB_70, + XTP_PCS_MODE_MASK | XTP_PCS_UPDT, + XTP_PCS_MODE(2) | XTP_PCS_UPDT); + } + usleep_range(1, 5); + + mtk_phy_clear_bits(xfi_tphy->base + REG_DIG_GLB_70, XTP_PCS_UPDT); + + usleep_range(100, 500); + + /* Enable MAC CK */ + mtk_phy_set_bits(xfi_tphy->base + REG_DIG_LN_TRX_B0, XTP_LN_TX_MACCK_EN); + mtk_phy_clear_bits(xfi_tphy->base + REG_DIG_GLB_F4, XFI_DPHY_AD_SGDT_FRC_EN); + + /* Enable TX data */ + mtk_phy_set_bits(xfi_tphy->base + REG_DIG_LN_TRX_40, + XTP_LN_FRC_TX_DATA_EN | XTP_LN_TX_DATA_EN); + usleep_range(400, 1000); +} + +/** + * mtk_xfi_tphy_set_mode() - Setup phy for specified interface mode. + * + * @phy: Phy instance. + * @mode: Only PHY_MODE_ETHERNET is supported. + * @submode: An Ethernet interface mode. + * + * Validate selected mode and call function mtk_xfi_tphy_setup(). + * + * Return: + * * %0 - OK + * * %-EINVAL - invalid mode + */ +static int mtk_xfi_tphy_set_mode(struct phy *phy, enum phy_mode mode, int + submode) +{ + struct mtk_xfi_tphy *xfi_tphy = phy_get_drvdata(phy); + + if (mode != PHY_MODE_ETHERNET) + return -EINVAL; + + switch (submode) { + case PHY_INTERFACE_MODE_1000BASEX: + case PHY_INTERFACE_MODE_2500BASEX: + case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_5GBASER: + case PHY_INTERFACE_MODE_10GBASER: + case PHY_INTERFACE_MODE_USXGMII: + mtk_xfi_tphy_setup(xfi_tphy, submode); + return 0; + default: + return -EINVAL; + } +} + +/** + * mtk_xfi_tphy_reset() - Reset the phy. + * + * @phy: Phy instance. + * + * Reset the phy using the external reset controller. + * + * Return: + * %0 - OK + */ +static int mtk_xfi_tphy_reset(struct phy *phy) +{ + struct mtk_xfi_tphy *xfi_tphy = phy_get_drvdata(phy); + + reset_control_assert(xfi_tphy->reset); + usleep_range(100, 500); + reset_control_deassert(xfi_tphy->reset); + usleep_range(1, 10); + + return 0; +} + +/** + * mtk_xfi_tphy_power_on() - Power-on the phy. + * + * @phy: Phy instance. + * + * Prepare and enable all clocks required for the phy to operate. + * + * Return: + * See clk_bulk_prepare_enable(). + */ +static int mtk_xfi_tphy_power_on(struct phy *phy) +{ + struct mtk_xfi_tphy *xfi_tphy = phy_get_drvdata(phy); + + return clk_bulk_prepare_enable(MTK_XFI_TPHY_NUM_CLOCKS, xfi_tphy->clocks); +} + +/** + * mtk_xfi_tphy_power_off() - Power-off the phy. + * + * @phy: Phy instance. + * + * Disable and unprepare all clocks previously enabled. + * + * Return: + * See clk_bulk_prepare_disable(). + */ +static int mtk_xfi_tphy_power_off(struct phy *phy) +{ + struct mtk_xfi_tphy *xfi_tphy = phy_get_drvdata(phy); + + clk_bulk_disable_unprepare(MTK_XFI_TPHY_NUM_CLOCKS, xfi_tphy->clocks); + + return 0; +} + +static const struct phy_ops mtk_xfi_tphy_ops = { + .power_on = mtk_xfi_tphy_power_on, + .power_off = mtk_xfi_tphy_power_off, + .set_mode = mtk_xfi_tphy_set_mode, + .reset = mtk_xfi_tphy_reset, + .owner = THIS_MODULE, +}; + +/** + * mtk_xfi_tphy_probe() - Probe phy instance from Device Tree. + * @pdev: Matching platform device. + * + * The probe function gets IO resource, clocks, reset controller and + * whether the DA work-around for 10GBase-R is required from Device Tree and + * allocates memory for holding that information in a struct mtk_xfi_tphy. + * + * Return: + * * %0 - OK + * * %-ENODEV - Missing associated Device Tree node (should never happen). + * * %-ENOMEM - Out of memory. + * * Any error value which devm_platform_ioremap_resource(), + * devm_clk_bulk_get(), devm_reset_control_get_exclusive(), + * devm_phy_create() or devm_of_phy_provider_register() may return. + */ +static int mtk_xfi_tphy_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct phy_provider *phy_provider; + struct mtk_xfi_tphy *xfi_tphy; + struct phy *phy; + int ret; + + if (!np) + return -ENODEV; + + xfi_tphy = devm_kzalloc(&pdev->dev, sizeof(*xfi_tphy), GFP_KERNEL); + if (!xfi_tphy) + return -ENOMEM; + + xfi_tphy->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(xfi_tphy->base)) + return PTR_ERR(xfi_tphy->base); + + xfi_tphy->dev = &pdev->dev; + xfi_tphy->clocks[0].id = "topxtal"; + xfi_tphy->clocks[1].id = "xfipll"; + ret = devm_clk_bulk_get(&pdev->dev, MTK_XFI_TPHY_NUM_CLOCKS, xfi_tphy->clocks); + if (ret) + return ret; + + xfi_tphy->reset = devm_reset_control_get_exclusive(&pdev->dev, NULL); + if (IS_ERR(xfi_tphy->reset)) + return PTR_ERR(xfi_tphy->reset); + + xfi_tphy->da_war = of_property_read_bool(np, "mediatek,usxgmii-performance-errata"); + + phy = devm_phy_create(&pdev->dev, NULL, &mtk_xfi_tphy_ops); + if (IS_ERR(phy)) + return PTR_ERR(phy); + + phy_set_drvdata(phy, xfi_tphy); + phy_provider = devm_of_phy_provider_register(&pdev->dev, of_phy_simple_xlate); + + return PTR_ERR_OR_ZERO(phy_provider); +} + +static const struct of_device_id mtk_xfi_tphy_match[] = { + { .compatible = "mediatek,mt7988-xfi-tphy", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, mtk_xfi_tphy_match); + +static struct platform_driver mtk_xfi_tphy_driver = { + .probe = mtk_xfi_tphy_probe, + .driver = { + .name = "mtk-xfi-tphy", + .of_match_table = mtk_xfi_tphy_match, + }, +}; +module_platform_driver(mtk_xfi_tphy_driver); + +MODULE_DESCRIPTION("MediaTek 10GE SerDes XFI T-PHY driver"); +MODULE_AUTHOR("Daniel Golle <daniel@makrotopia.org>"); +MODULE_AUTHOR("Bc-bocun Chen <bc-bocun.chen@mediatek.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c index c5c8d70bc853..bf6a07590321 100644 --- a/drivers/phy/phy-core.c +++ b/drivers/phy/phy-core.c @@ -20,7 +20,12 @@ #include <linux/pm_runtime.h> #include <linux/regulator/consumer.h> -static struct class *phy_class; +static void phy_release(struct device *dev); +static const struct class phy_class = { + .name = "phy", + .dev_release = phy_release, +}; + static struct dentry *phy_debugfs_root; static DEFINE_MUTEX(phy_provider_mutex); static LIST_HEAD(phy_provider_list); @@ -753,7 +758,7 @@ struct phy *of_phy_simple_xlate(struct device *dev, struct phy *phy; struct class_dev_iter iter; - class_dev_iter_init(&iter, phy_class, NULL, NULL); + class_dev_iter_init(&iter, &phy_class, NULL, NULL); while ((dev = class_dev_iter_next(&iter))) { phy = to_phy(dev); if (args->np != phy->dev.of_node) @@ -1016,7 +1021,7 @@ struct phy *phy_create(struct device *dev, struct device_node *node, device_initialize(&phy->dev); mutex_init(&phy->mutex); - phy->dev.class = phy_class; + phy->dev.class = &phy_class; phy->dev.parent = dev; phy->dev.of_node = node ?: dev->of_node; phy->id = id; @@ -1285,14 +1290,13 @@ static void phy_release(struct device *dev) static int __init phy_core_init(void) { - phy_class = class_create("phy"); - if (IS_ERR(phy_class)) { - pr_err("failed to create phy class --> %ld\n", - PTR_ERR(phy_class)); - return PTR_ERR(phy_class); - } + int err; - phy_class->dev_release = phy_release; + err = class_register(&phy_class); + if (err) { + pr_err("failed to register phy class"); + return err; + } phy_debugfs_root = debugfs_create_dir("phy", NULL); @@ -1303,6 +1307,6 @@ device_initcall(phy_core_init); static void __exit phy_core_exit(void) { debugfs_remove_recursive(phy_debugfs_root); - class_destroy(phy_class); + class_unregister(&phy_class); } module_exit(phy_core_exit); diff --git a/drivers/phy/qualcomm/phy-qcom-edp.c b/drivers/phy/qualcomm/phy-qcom-edp.c index 621d0453bf76..da2b32fb5b45 100644 --- a/drivers/phy/qualcomm/phy-qcom-edp.c +++ b/drivers/phy/qualcomm/phy-qcom-edp.c @@ -24,6 +24,7 @@ #include "phy-qcom-qmp-dp-phy.h" #include "phy-qcom-qmp-qserdes-com-v4.h" +#include "phy-qcom-qmp-qserdes-com-v6.h" /* EDP_PHY registers */ #define DP_PHY_CFG 0x0010 @@ -77,9 +78,20 @@ struct qcom_edp_swing_pre_emph_cfg { const u8 (*pre_emphasis_hbr3_hbr2)[4][4]; }; +struct qcom_edp; + +struct phy_ver_ops { + int (*com_power_on)(const struct qcom_edp *edp); + int (*com_resetsm_cntrl)(const struct qcom_edp *edp); + int (*com_bias_en_clkbuflr)(const struct qcom_edp *edp); + int (*com_configure_pll)(const struct qcom_edp *edp); + int (*com_configure_ssc)(const struct qcom_edp *edp); +}; + struct qcom_edp_phy_cfg { bool is_edp; const struct qcom_edp_swing_pre_emph_cfg *swing_pre_emph_cfg; + const struct phy_ver_ops *ver_ops; }; struct qcom_edp { @@ -174,18 +186,6 @@ static const struct qcom_edp_swing_pre_emph_cfg edp_phy_swing_pre_emph_cfg = { .pre_emphasis_hbr3_hbr2 = &edp_pre_emp_hbr2_hbr3, }; -static const struct qcom_edp_phy_cfg sc7280_dp_phy_cfg = { -}; - -static const struct qcom_edp_phy_cfg sc8280xp_dp_phy_cfg = { - .swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg, -}; - -static const struct qcom_edp_phy_cfg sc8280xp_edp_phy_cfg = { - .is_edp = true, - .swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg, -}; - static int qcom_edp_phy_init(struct phy *phy) { struct qcom_edp *edp = phy_get_drvdata(phy); @@ -204,8 +204,9 @@ static int qcom_edp_phy_init(struct phy *phy) DP_PHY_PD_CTL_PLL_PWRDN | DP_PHY_PD_CTL_DP_CLAMP_EN, edp->edp + DP_PHY_PD_CTL); - /* Turn on BIAS current for PHY/PLL */ - writel(0x17, edp->pll + QSERDES_V4_COM_BIAS_EN_CLKBUFLR_EN); + ret = edp->cfg->ver_ops->com_bias_en_clkbuflr(edp); + if (ret) + return ret; writel(DP_PHY_PD_CTL_PSR_PWRDN, edp->edp + DP_PHY_PD_CTL); msleep(20); @@ -313,6 +314,84 @@ static int qcom_edp_phy_configure(struct phy *phy, union phy_configure_opts *opt static int qcom_edp_configure_ssc(const struct qcom_edp *edp) { + return edp->cfg->ver_ops->com_configure_ssc(edp); +} + +static int qcom_edp_configure_pll(const struct qcom_edp *edp) +{ + return edp->cfg->ver_ops->com_configure_pll(edp); +} + +static int qcom_edp_set_vco_div(const struct qcom_edp *edp, unsigned long *pixel_freq) +{ + const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts; + u32 vco_div; + + switch (dp_opts->link_rate) { + case 1620: + vco_div = 0x1; + *pixel_freq = 1620000000UL / 2; + break; + + case 2700: + vco_div = 0x1; + *pixel_freq = 2700000000UL / 2; + break; + + case 5400: + vco_div = 0x2; + *pixel_freq = 5400000000UL / 4; + break; + + case 8100: + vco_div = 0x0; + *pixel_freq = 8100000000UL / 6; + break; + + default: + /* Other link rates aren't supported */ + return -EINVAL; + } + + writel(vco_div, edp->edp + DP_PHY_VCO_DIV); + + return 0; +} + +static int qcom_edp_phy_power_on_v4(const struct qcom_edp *edp) +{ + u32 val; + + writel(DP_PHY_PD_CTL_PWRDN | DP_PHY_PD_CTL_AUX_PWRDN | + DP_PHY_PD_CTL_LANE_0_1_PWRDN | DP_PHY_PD_CTL_LANE_2_3_PWRDN | + DP_PHY_PD_CTL_PLL_PWRDN | DP_PHY_PD_CTL_DP_CLAMP_EN, + edp->edp + DP_PHY_PD_CTL); + writel(0xfc, edp->edp + DP_PHY_MODE); + + return readl_poll_timeout(edp->pll + QSERDES_V4_COM_CMN_STATUS, + val, val & BIT(7), 5, 200); +} + +static int qcom_edp_phy_com_resetsm_cntrl_v4(const struct qcom_edp *edp) +{ + u32 val; + + writel(0x20, edp->pll + QSERDES_V4_COM_RESETSM_CNTRL); + + return readl_poll_timeout(edp->pll + QSERDES_V4_COM_C_READY_STATUS, + val, val & BIT(0), 500, 10000); +} + +static int qcom_edp_com_bias_en_clkbuflr_v4(const struct qcom_edp *edp) +{ + /* Turn on BIAS current for PHY/PLL */ + writel(0x17, edp->pll + QSERDES_V4_COM_BIAS_EN_CLKBUFLR_EN); + + return 0; +} + +static int qcom_edp_com_configure_ssc_v4(const struct qcom_edp *edp) +{ const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts; u32 step1; u32 step2; @@ -345,7 +424,7 @@ static int qcom_edp_configure_ssc(const struct qcom_edp *edp) return 0; } -static int qcom_edp_configure_pll(const struct qcom_edp *edp) +static int qcom_edp_com_configure_pll_v4(const struct qcom_edp *edp) { const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts; u32 div_frac_start2_mode0; @@ -431,30 +510,150 @@ static int qcom_edp_configure_pll(const struct qcom_edp *edp) return 0; } -static int qcom_edp_set_vco_div(const struct qcom_edp *edp, unsigned long *pixel_freq) +static const struct phy_ver_ops qcom_edp_phy_ops_v4 = { + .com_power_on = qcom_edp_phy_power_on_v4, + .com_resetsm_cntrl = qcom_edp_phy_com_resetsm_cntrl_v4, + .com_bias_en_clkbuflr = qcom_edp_com_bias_en_clkbuflr_v4, + .com_configure_pll = qcom_edp_com_configure_pll_v4, + .com_configure_ssc = qcom_edp_com_configure_ssc_v4, +}; + +static const struct qcom_edp_phy_cfg sc7280_dp_phy_cfg = { + .ver_ops = &qcom_edp_phy_ops_v4, +}; + +static const struct qcom_edp_phy_cfg sc8280xp_dp_phy_cfg = { + .swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg, + .ver_ops = &qcom_edp_phy_ops_v4, +}; + +static const struct qcom_edp_phy_cfg sc8280xp_edp_phy_cfg = { + .is_edp = true, + .swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg, + .ver_ops = &qcom_edp_phy_ops_v4, +}; + +static int qcom_edp_phy_power_on_v6(const struct qcom_edp *edp) +{ + u32 val; + + writel(DP_PHY_PD_CTL_PWRDN | DP_PHY_PD_CTL_AUX_PWRDN | + DP_PHY_PD_CTL_LANE_0_1_PWRDN | DP_PHY_PD_CTL_LANE_2_3_PWRDN | + DP_PHY_PD_CTL_PLL_PWRDN | DP_PHY_PD_CTL_DP_CLAMP_EN, + edp->edp + DP_PHY_PD_CTL); + writel(0xfc, edp->edp + DP_PHY_MODE); + + return readl_poll_timeout(edp->pll + QSERDES_V6_COM_CMN_STATUS, + val, val & BIT(7), 5, 200); +} + +static int qcom_edp_phy_com_resetsm_cntrl_v6(const struct qcom_edp *edp) +{ + u32 val; + + writel(0x20, edp->pll + QSERDES_V6_COM_RESETSM_CNTRL); + + return readl_poll_timeout(edp->pll + QSERDES_V6_COM_C_READY_STATUS, + val, val & BIT(0), 500, 10000); +} + +static int qcom_edp_com_bias_en_clkbuflr_v6(const struct qcom_edp *edp) +{ + /* Turn on BIAS current for PHY/PLL */ + writel(0x1f, edp->pll + QSERDES_V6_COM_PLL_BIAS_EN_CLK_BUFLR_EN); + + return 0; +} + +static int qcom_edp_com_configure_ssc_v6(const struct qcom_edp *edp) { const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts; - u32 vco_div; + u32 step1; + u32 step2; switch (dp_opts->link_rate) { case 1620: - vco_div = 0x1; - *pixel_freq = 1620000000UL / 2; + case 2700: + case 8100: + step1 = 0x92; + step2 = 0x01; + break; + + case 5400: + step1 = 0x18; + step2 = 0x02; + break; + + default: + /* Other link rates aren't supported */ + return -EINVAL; + } + + writel(0x01, edp->pll + QSERDES_V6_COM_SSC_EN_CENTER); + writel(0x00, edp->pll + QSERDES_V6_COM_SSC_ADJ_PER1); + writel(0x36, edp->pll + QSERDES_V6_COM_SSC_PER1); + writel(0x01, edp->pll + QSERDES_V6_COM_SSC_PER2); + writel(step1, edp->pll + QSERDES_V6_COM_SSC_STEP_SIZE1_MODE0); + writel(step2, edp->pll + QSERDES_V6_COM_SSC_STEP_SIZE2_MODE0); + + return 0; +} + +static int qcom_edp_com_configure_pll_v6(const struct qcom_edp *edp) +{ + const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts; + u32 div_frac_start2_mode0; + u32 div_frac_start3_mode0; + u32 dec_start_mode0; + u32 lock_cmp1_mode0; + u32 lock_cmp2_mode0; + u32 code1_mode0; + u32 code2_mode0; + u32 hsclk_sel; + + switch (dp_opts->link_rate) { + case 1620: + hsclk_sel = 0x5; + dec_start_mode0 = 0x34; + div_frac_start2_mode0 = 0xc0; + div_frac_start3_mode0 = 0x0b; + lock_cmp1_mode0 = 0x37; + lock_cmp2_mode0 = 0x04; + code1_mode0 = 0x71; + code2_mode0 = 0x0c; break; case 2700: - vco_div = 0x1; - *pixel_freq = 2700000000UL / 2; + hsclk_sel = 0x3; + dec_start_mode0 = 0x34; + div_frac_start2_mode0 = 0xc0; + div_frac_start3_mode0 = 0x0b; + lock_cmp1_mode0 = 0x07; + lock_cmp2_mode0 = 0x07; + code1_mode0 = 0x71; + code2_mode0 = 0x0c; break; case 5400: - vco_div = 0x2; - *pixel_freq = 5400000000UL / 4; + hsclk_sel = 0x1; + dec_start_mode0 = 0x46; + div_frac_start2_mode0 = 0x00; + div_frac_start3_mode0 = 0x05; + lock_cmp1_mode0 = 0x0f; + lock_cmp2_mode0 = 0x0e; + code1_mode0 = 0x97; + code2_mode0 = 0x10; break; case 8100: - vco_div = 0x0; - *pixel_freq = 8100000000UL / 6; + hsclk_sel = 0x0; + dec_start_mode0 = 0x34; + div_frac_start2_mode0 = 0xc0; + div_frac_start3_mode0 = 0x0b; + lock_cmp1_mode0 = 0x17; + lock_cmp2_mode0 = 0x15; + code1_mode0 = 0x71; + code2_mode0 = 0x0c; break; default: @@ -462,33 +661,69 @@ static int qcom_edp_set_vco_div(const struct qcom_edp *edp, unsigned long *pixel return -EINVAL; } - writel(vco_div, edp->edp + DP_PHY_VCO_DIV); + writel(0x01, edp->pll + QSERDES_V6_COM_SVS_MODE_CLK_SEL); + writel(0x0b, edp->pll + QSERDES_V6_COM_SYSCLK_EN_SEL); + writel(0x02, edp->pll + QSERDES_V6_COM_SYS_CLK_CTRL); + writel(0x0c, edp->pll + QSERDES_V6_COM_CLK_ENABLE1); + writel(0x06, edp->pll + QSERDES_V6_COM_SYSCLK_BUF_ENABLE); + writel(0x30, edp->pll + QSERDES_V6_COM_CLK_SELECT); + writel(hsclk_sel, edp->pll + QSERDES_V6_COM_HSCLK_SEL_1); + writel(0x07, edp->pll + QSERDES_V6_COM_PLL_IVCO); + writel(0x08, edp->pll + QSERDES_V6_COM_LOCK_CMP_EN); + writel(0x36, edp->pll + QSERDES_V6_COM_PLL_CCTRL_MODE0); + writel(0x16, edp->pll + QSERDES_V6_COM_PLL_RCTRL_MODE0); + writel(0x06, edp->pll + QSERDES_V6_COM_CP_CTRL_MODE0); + writel(dec_start_mode0, edp->pll + QSERDES_V6_COM_DEC_START_MODE0); + writel(0x00, edp->pll + QSERDES_V6_COM_DIV_FRAC_START1_MODE0); + writel(div_frac_start2_mode0, edp->pll + QSERDES_V6_COM_DIV_FRAC_START2_MODE0); + writel(div_frac_start3_mode0, edp->pll + QSERDES_V6_COM_DIV_FRAC_START3_MODE0); + writel(0x12, edp->pll + QSERDES_V6_COM_CMN_CONFIG_1); + writel(0x3f, edp->pll + QSERDES_V6_COM_INTEGLOOP_GAIN0_MODE0); + writel(0x00, edp->pll + QSERDES_V6_COM_INTEGLOOP_GAIN1_MODE0); + writel(0x00, edp->pll + QSERDES_V6_COM_VCO_TUNE_MAP); + writel(lock_cmp1_mode0, edp->pll + QSERDES_V6_COM_LOCK_CMP1_MODE0); + writel(lock_cmp2_mode0, edp->pll + QSERDES_V6_COM_LOCK_CMP2_MODE0); + + writel(0x0a, edp->pll + QSERDES_V6_COM_BG_TIMER); + writel(0x14, edp->pll + QSERDES_V6_COM_PLL_CORE_CLK_DIV_MODE0); + writel(0x00, edp->pll + QSERDES_V6_COM_VCO_TUNE_CTRL); + writel(0x1f, edp->pll + QSERDES_V6_COM_PLL_BIAS_EN_CLK_BUFLR_EN); + writel(0x0f, edp->pll + QSERDES_V6_COM_CORE_CLK_EN); + writel(0xa0, edp->pll + QSERDES_V6_COM_VCO_TUNE1_MODE0); + writel(0x03, edp->pll + QSERDES_V6_COM_VCO_TUNE2_MODE0); + + writel(code1_mode0, edp->pll + QSERDES_V6_COM_BIN_VCOCAL_CMP_CODE1_MODE0); + writel(code2_mode0, edp->pll + QSERDES_V6_COM_BIN_VCOCAL_CMP_CODE2_MODE0); return 0; } +static const struct phy_ver_ops qcom_edp_phy_ops_v6 = { + .com_power_on = qcom_edp_phy_power_on_v6, + .com_resetsm_cntrl = qcom_edp_phy_com_resetsm_cntrl_v6, + .com_bias_en_clkbuflr = qcom_edp_com_bias_en_clkbuflr_v6, + .com_configure_pll = qcom_edp_com_configure_pll_v6, + .com_configure_ssc = qcom_edp_com_configure_ssc_v6, +}; + +static struct qcom_edp_phy_cfg x1e80100_phy_cfg = { + .swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg, + .ver_ops = &qcom_edp_phy_ops_v6, +}; + static int qcom_edp_phy_power_on(struct phy *phy) { const struct qcom_edp *edp = phy_get_drvdata(phy); u32 bias0_en, drvr0_en, bias1_en, drvr1_en; unsigned long pixel_freq; u8 ldo_config = 0x0; - int timeout; int ret; u32 val; u8 cfg1; - writel(DP_PHY_PD_CTL_PWRDN | DP_PHY_PD_CTL_AUX_PWRDN | - DP_PHY_PD_CTL_LANE_0_1_PWRDN | DP_PHY_PD_CTL_LANE_2_3_PWRDN | - DP_PHY_PD_CTL_PLL_PWRDN | DP_PHY_PD_CTL_DP_CLAMP_EN, - edp->edp + DP_PHY_PD_CTL); - writel(0xfc, edp->edp + DP_PHY_MODE); - - timeout = readl_poll_timeout(edp->pll + QSERDES_V4_COM_CMN_STATUS, - val, val & BIT(7), 5, 200); - if (timeout) - return timeout; - + ret = edp->cfg->ver_ops->com_power_on(edp); + if (ret) + return ret; if (edp->cfg->swing_pre_emph_cfg && !edp->is_edp) ldo_config = 0x1; @@ -535,12 +770,9 @@ static int qcom_edp_phy_power_on(struct phy *phy) writel(0x01, edp->edp + DP_PHY_CFG); writel(0x09, edp->edp + DP_PHY_CFG); - writel(0x20, edp->pll + QSERDES_V4_COM_RESETSM_CNTRL); - - timeout = readl_poll_timeout(edp->pll + QSERDES_V4_COM_C_READY_STATUS, - val, val & BIT(0), 500, 10000); - if (timeout) - return timeout; + ret = edp->cfg->ver_ops->com_resetsm_cntrl(edp); + if (ret) + return ret; writel(0x19, edp->edp + DP_PHY_CFG); writel(0x1f, edp->tx0 + TXn_HIGHZ_DRVR_EN); @@ -880,6 +1112,7 @@ static const struct of_device_id qcom_edp_phy_match_table[] = { { .compatible = "qcom,sc8180x-edp-phy", .data = &sc7280_dp_phy_cfg, }, { .compatible = "qcom,sc8280xp-dp-phy", .data = &sc8280xp_dp_phy_cfg, }, { .compatible = "qcom,sc8280xp-edp-phy", .data = &sc8280xp_edp_phy_cfg, }, + { .compatible = "qcom,x1e80100-dp-phy", .data = &x1e80100_phy_cfg, }, { } }; MODULE_DEVICE_TABLE(of, qcom_edp_phy_match_table); diff --git a/drivers/phy/qualcomm/phy-qcom-eusb2-repeater.c b/drivers/phy/qualcomm/phy-qcom-eusb2-repeater.c index a43e20abb10d..68cc8e24f383 100644 --- a/drivers/phy/qualcomm/phy-qcom-eusb2-repeater.c +++ b/drivers/phy/qualcomm/phy-qcom-eusb2-repeater.c @@ -88,6 +88,12 @@ static const u32 pm8550b_init_tbl[NUM_TUNE_FIELDS] = { [TUNE_USB2_PREEM] = 0x5, }; +static const u32 smb2360_init_tbl[NUM_TUNE_FIELDS] = { + [TUNE_IUSB2] = 0x5, + [TUNE_SQUELCH_U] = 0x3, + [TUNE_USB2_PREEM] = 0x2, +}; + static const struct eusb2_repeater_cfg pm8550b_eusb2_cfg = { .init_tbl = pm8550b_init_tbl, .init_tbl_num = ARRAY_SIZE(pm8550b_init_tbl), @@ -95,6 +101,13 @@ static const struct eusb2_repeater_cfg pm8550b_eusb2_cfg = { .num_vregs = ARRAY_SIZE(pm8550b_vreg_l), }; +static const struct eusb2_repeater_cfg smb2360_eusb2_cfg = { + .init_tbl = smb2360_init_tbl, + .init_tbl_num = ARRAY_SIZE(smb2360_init_tbl), + .vreg_list = pm8550b_vreg_l, + .num_vregs = ARRAY_SIZE(pm8550b_vreg_l), +}; + static int eusb2_repeater_init_vregs(struct eusb2_repeater *rptr) { int num = rptr->cfg->num_vregs; @@ -271,6 +284,10 @@ static const struct of_device_id eusb2_repeater_of_match_table[] = { .compatible = "qcom,pm8550b-eusb2-repeater", .data = &pm8550b_eusb2_cfg, }, + { + .compatible = "qcom,smb2360-eusb2-repeater", + .data = &smb2360_eusb2_cfg, + }, { }, }; MODULE_DEVICE_TABLE(of, eusb2_repeater_of_match_table); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c index c21cdb8dbfe7..7f999e8a433d 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c @@ -1382,6 +1382,13 @@ static const u8 qmp_dp_v5_voltage_swing_hbr_rbr[4][4] = { { 0x3f, 0xff, 0xff, 0xff } }; +static const u8 qmp_dp_v6_voltage_swing_hbr_rbr[4][4] = { + { 0x27, 0x2f, 0x36, 0x3f }, + { 0x31, 0x3e, 0x3f, 0xff }, + { 0x36, 0x3f, 0xff, 0xff }, + { 0x3f, 0xff, 0xff, 0xff } +}; + static const u8 qmp_dp_v6_pre_emphasis_hbr_rbr[4][4] = { { 0x20, 0x2d, 0x34, 0x3a }, { 0x20, 0x2e, 0x35, 0xff }, @@ -2001,6 +2008,51 @@ static const struct qmp_phy_cfg sm8550_usb3dpphy_cfg = { .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), }; +static const struct qmp_phy_cfg sm8650_usb3dpphy_cfg = { + .offsets = &qmp_combo_offsets_v3, + + .serdes_tbl = sm8550_usb3_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sm8550_usb3_serdes_tbl), + .tx_tbl = sm8550_usb3_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sm8550_usb3_tx_tbl), + .rx_tbl = sm8550_usb3_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sm8550_usb3_rx_tbl), + .pcs_tbl = sm8550_usb3_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sm8550_usb3_pcs_tbl), + .pcs_usb_tbl = sm8550_usb3_pcs_usb_tbl, + .pcs_usb_tbl_num = ARRAY_SIZE(sm8550_usb3_pcs_usb_tbl), + + .dp_serdes_tbl = qmp_v6_dp_serdes_tbl, + .dp_serdes_tbl_num = ARRAY_SIZE(qmp_v6_dp_serdes_tbl), + .dp_tx_tbl = qmp_v6_dp_tx_tbl, + .dp_tx_tbl_num = ARRAY_SIZE(qmp_v6_dp_tx_tbl), + + .serdes_tbl_rbr = qmp_v6_dp_serdes_tbl_rbr, + .serdes_tbl_rbr_num = ARRAY_SIZE(qmp_v6_dp_serdes_tbl_rbr), + .serdes_tbl_hbr = qmp_v6_dp_serdes_tbl_hbr, + .serdes_tbl_hbr_num = ARRAY_SIZE(qmp_v6_dp_serdes_tbl_hbr), + .serdes_tbl_hbr2 = qmp_v6_dp_serdes_tbl_hbr2, + .serdes_tbl_hbr2_num = ARRAY_SIZE(qmp_v6_dp_serdes_tbl_hbr2), + .serdes_tbl_hbr3 = qmp_v6_dp_serdes_tbl_hbr3, + .serdes_tbl_hbr3_num = ARRAY_SIZE(qmp_v6_dp_serdes_tbl_hbr3), + + .swing_hbr_rbr = &qmp_dp_v6_voltage_swing_hbr_rbr, + .pre_emphasis_hbr_rbr = &qmp_dp_v6_pre_emphasis_hbr_rbr, + .swing_hbr3_hbr2 = &qmp_dp_v5_voltage_swing_hbr3_hbr2, + .pre_emphasis_hbr3_hbr2 = &qmp_dp_v5_pre_emphasis_hbr3_hbr2, + + .dp_aux_init = qmp_v4_dp_aux_init, + .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_phy = qmp_v4_configure_dp_phy, + .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, + + .regs = qmp_v6_usb3phy_regs_layout, + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), +}; + static int qmp_combo_dp_serdes_init(struct qmp_combo *qmp) { const struct qmp_phy_cfg *cfg = qmp->cfg; @@ -2437,8 +2489,6 @@ static int qmp_v4_configure_dp_phy(struct qmp_combo *qmp) writel(0x20, qmp->dp_tx2 + cfg->regs[QPHY_TX_TX_EMP_POST1_LVL]); return 0; - - return 0; } /* @@ -3631,7 +3681,7 @@ static const struct of_device_id qmp_combo_of_match_table[] = { }, { .compatible = "qcom,sm8650-qmp-usb3-dp-phy", - .data = &sm8550_usb3dpphy_cfg, + .data = &sm8650_usb3dpphy_cfg, }, { .compatible = "qcom,x1e80100-qmp-usb3-dp-phy", diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c index 8836bb1ff0cc..6c796723c8f5 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c @@ -22,6 +22,8 @@ #include <linux/reset.h> #include <linux/slab.h> +#include <dt-bindings/phy/phy-qcom-qmp.h> + #include "phy-qcom-qmp-common.h" #include "phy-qcom-qmp.h" @@ -2246,6 +2248,7 @@ static const struct qmp_phy_init_tbl sa8775p_qmp_gen4x4_pcie_pcs_alt_tbl[] = { }; static const struct qmp_phy_init_tbl sa8775p_qmp_gen4x4_pcie_serdes_alt_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIAS_EN_CLKBUFLR_EN, 0x1c), QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_IVCO, 0x0f), QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_CCTRL_MODE0, 0x36), QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_CCTRL_MODE1, 0x36), @@ -2272,7 +2275,6 @@ static const struct qmp_phy_init_tbl sa8775p_qmp_gen4x4_pcie_rc_serdes_alt_tbl[] QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE2_MODE0, 0x07), QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE1_MODE1, 0x97), QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE2_MODE1, 0x0c), - QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIAS_EN_CLKBUFLR_EN, 0x1c), QMP_PHY_INIT_CFG(QSERDES_V5_COM_CLK_ENABLE1, 0x90), QMP_PHY_INIT_CFG(QSERDES_V5_COM_CP_CTRL_MODE0, 0x06), QMP_PHY_INIT_CFG(QSERDES_V5_COM_CP_CTRL_MODE1, 0x06), @@ -2389,6 +2391,9 @@ struct qmp_phy_cfg { /* QMP PHY pipe clock interface rate */ unsigned long pipe_clock_rate; + + /* QMP PHY AUX clock interface rate */ + unsigned long aux_clock_rate; }; struct qmp_pcie { @@ -2420,6 +2425,7 @@ struct qmp_pcie { int mode; struct clk_fixed_rate pipe_clk_fixed; + struct clk_fixed_rate aux_clk_fixed; }; static inline void qphy_setbits(void __iomem *base, u32 offset, u32 val) @@ -3135,6 +3141,9 @@ static const struct qmp_phy_cfg sm8450_qmp_gen4x2_pciephy_cfg = { .pwrdn_ctrl = SW_PWRDN | REFCLK_DRV_DSBL, .phy_status = PHYSTATUS_4_20, + + /* 20MHz PHY AUX Clock */ + .aux_clock_rate = 20000000, }; static const struct qmp_phy_cfg sm8550_qmp_gen3x2_pciephy_cfg = { @@ -3192,6 +3201,9 @@ static const struct qmp_phy_cfg sm8550_qmp_gen4x2_pciephy_cfg = { .pwrdn_ctrl = SW_PWRDN | REFCLK_DRV_DSBL, .phy_status = PHYSTATUS_4_20, .has_nocsr_reset = true, + + /* 20MHz PHY AUX Clock */ + .aux_clock_rate = 20000000, }; static const struct qmp_phy_cfg sm8650_qmp_gen4x2_pciephy_cfg = { @@ -3222,6 +3234,9 @@ static const struct qmp_phy_cfg sm8650_qmp_gen4x2_pciephy_cfg = { .pwrdn_ctrl = SW_PWRDN | REFCLK_DRV_DSBL, .phy_status = PHYSTATUS_4_20, .has_nocsr_reset = true, + + /* 20MHz PHY AUX Clock */ + .aux_clock_rate = 20000000, }; static const struct qmp_phy_cfg sa8775p_qmp_gen4x2_pciephy_cfg = { @@ -3291,6 +3306,13 @@ static const struct qmp_phy_cfg sa8775p_qmp_gen4x4_pciephy_cfg = { .pcs_misc_num = ARRAY_SIZE(sa8775p_qmp_gen4_pcie_rc_pcs_misc_tbl), }, + .tbls_ep = &(const struct qmp_phy_cfg_tbls) { + .serdes = sa8775p_qmp_gen4x2_pcie_ep_serdes_alt_tbl, + .serdes_num = ARRAY_SIZE(sa8775p_qmp_gen4x2_pcie_ep_serdes_alt_tbl), + .pcs_misc = sm8450_qmp_gen4x2_pcie_ep_pcs_misc_tbl, + .pcs_misc_num = ARRAY_SIZE(sm8450_qmp_gen4x2_pcie_ep_pcs_misc_tbl), + }, + .reset_list = sdm845_pciephy_reset_l, .num_resets = ARRAY_SIZE(sdm845_pciephy_reset_l), .vreg_list = qmp_phy_vreg_l, @@ -3664,7 +3686,7 @@ static int phy_pipe_clk_register(struct qmp_pcie *qmp, struct device_node *np) struct clk_init_data init = { }; int ret; - ret = of_property_read_string(np, "clock-output-names", &init.name); + ret = of_property_read_string_index(np, "clock-output-names", 0, &init.name); if (ret) { dev_err(qmp->dev, "%pOFn: No clock-output-names\n", np); return ret; @@ -3683,14 +3705,87 @@ static int phy_pipe_clk_register(struct qmp_pcie *qmp, struct device_node *np) fixed->hw.init = &init; - ret = devm_clk_hw_register(qmp->dev, &fixed->hw); - if (ret) + return devm_clk_hw_register(qmp->dev, &fixed->hw); +} + +/* + * Register a fixed rate PHY aux clock. + * + * The <s>_phy_aux_clksrc generated by PHY goes to the GCC that gate + * controls it. The <s>_phy_aux_clk coming out of the GCC is requested + * by the PHY driver for its operations. + * We register the <s>_phy_aux_clksrc here. The gcc driver takes care + * of assigning this <s>_phy_aux_clksrc as parent to <s>_phy_aux_clk. + * Below picture shows this relationship. + * + * +---------------+ + * | PHY block |<<---------------------------------------------+ + * | | | + * | +-------+ | +-----+ | + * I/P---^-->| PLL |---^--->phy_aux_clksrc--->| GCC |--->phy_aux_clk---+ + * clk | +-------+ | +-----+ + * +---------------+ + */ +static int phy_aux_clk_register(struct qmp_pcie *qmp, struct device_node *np) +{ + struct clk_fixed_rate *fixed = &qmp->aux_clk_fixed; + struct clk_init_data init = { }; + int ret; + + ret = of_property_read_string_index(np, "clock-output-names", 1, &init.name); + if (ret) { + dev_err(qmp->dev, "%pOFn: No clock-output-names index 1\n", np); return ret; + } + + init.ops = &clk_fixed_rate_ops; + + fixed->fixed_rate = qmp->cfg->aux_clock_rate; + fixed->hw.init = &init; + + return devm_clk_hw_register(qmp->dev, &fixed->hw); +} + +static struct clk_hw *qmp_pcie_clk_hw_get(struct of_phandle_args *clkspec, void *data) +{ + struct qmp_pcie *qmp = data; + + /* Support legacy bindings */ + if (!clkspec->args_count) + return &qmp->pipe_clk_fixed.hw; - ret = of_clk_add_hw_provider(np, of_clk_hw_simple_get, &fixed->hw); + switch (clkspec->args[0]) { + case QMP_PCIE_PIPE_CLK: + return &qmp->pipe_clk_fixed.hw; + case QMP_PCIE_PHY_AUX_CLK: + return &qmp->aux_clk_fixed.hw; + } + + return ERR_PTR(-EINVAL); +} + +static int qmp_pcie_register_clocks(struct qmp_pcie *qmp, struct device_node *np) +{ + int ret; + + ret = phy_pipe_clk_register(qmp, np); if (ret) return ret; + if (qmp->cfg->aux_clock_rate) { + ret = phy_aux_clk_register(qmp, np); + if (ret) + return ret; + + ret = of_clk_add_hw_provider(np, qmp_pcie_clk_hw_get, qmp); + if (ret) + return ret; + } else { + ret = of_clk_add_hw_provider(np, of_clk_hw_simple_get, &qmp->pipe_clk_fixed.hw); + if (ret) + return ret; + } + /* * Roll a devm action because the clock provider is the child node, but * the child node is not actually a device. @@ -3899,7 +3994,7 @@ static int qmp_pcie_probe(struct platform_device *pdev) if (ret) goto err_node_put; - ret = phy_pipe_clk_register(qmp, np); + ret = qmp_pcie_register_clocks(qmp, np); if (ret) goto err_node_put; diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-ufs-v6.h b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-ufs-v6.h index 970cc0667809..f19f9892ed7b 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-ufs-v6.h +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-ufs-v6.h @@ -30,5 +30,9 @@ #define QPHY_V6_PCS_UFS_TX_MID_TERM_CTRL1 0x1f4 #define QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1 0x1fc #define QPHY_V6_PCS_UFS_RX_HSG5_SYNC_WAIT_TIME 0x220 +#define QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S4 0x240 +#define QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S5 0x244 +#define QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S6 0x248 +#define QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S7 0x24c #endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-ufs-v6.h b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-ufs-v6.h index d9a87bd95590..d17a52357965 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-ufs-v6.h +++ b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-ufs-v6.h @@ -25,12 +25,15 @@ #define QSERDES_UFS_V6_RX_UCDR_SO_GAIN_RATE4 0xf0 #define QSERDES_UFS_V6_RX_UCDR_PI_CONTROLS 0xf4 #define QSERDES_UFS_V6_RX_VGA_CAL_MAN_VAL 0x178 +#define QSERDES_UFS_V6_RX_RX_EQU_ADAPTOR_CNTRL4 0x1ac #define QSERDES_UFS_V6_RX_EQ_OFFSET_ADAPTOR_CNTRL1 0x1bc #define QSERDES_UFS_V6_RX_INTERFACE_MODE 0x1e0 #define QSERDES_UFS_V6_RX_OFFSET_ADAPTOR_CNTRL3 0x1c4 #define QSERDES_UFS_V6_RX_MODE_RATE_0_1_B0 0x208 #define QSERDES_UFS_V6_RX_MODE_RATE_0_1_B1 0x20c +#define QSERDES_UFS_V6_RX_MODE_RATE_0_1_B2 0x210 #define QSERDES_UFS_V6_RX_MODE_RATE_0_1_B3 0x214 +#define QSERDES_UFS_V6_RX_MODE_RATE_0_1_B4 0x218 #define QSERDES_UFS_V6_RX_MODE_RATE_0_1_B6 0x220 #define QSERDES_UFS_V6_RX_MODE_RATE2_B3 0x238 #define QSERDES_UFS_V6_RX_MODE_RATE2_B6 0x244 @@ -38,6 +41,9 @@ #define QSERDES_UFS_V6_RX_MODE_RATE3_B4 0x260 #define QSERDES_UFS_V6_RX_MODE_RATE3_B5 0x264 #define QSERDES_UFS_V6_RX_MODE_RATE3_B8 0x270 +#define QSERDES_UFS_V6_RX_MODE_RATE4_B0 0x274 +#define QSERDES_UFS_V6_RX_MODE_RATE4_B1 0x278 +#define QSERDES_UFS_V6_RX_MODE_RATE4_B2 0x27c #define QSERDES_UFS_V6_RX_MODE_RATE4_B3 0x280 #define QSERDES_UFS_V6_RX_MODE_RATE4_B4 0x284 #define QSERDES_UFS_V6_RX_MODE_RATE4_B6 0x28c diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c index 590432d581f9..a57e8a4657f4 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c @@ -722,6 +722,38 @@ static const struct qmp_phy_init_tbl sm8350_ufsphy_g4_pcs[] = { QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_BIST_FIXED_PAT_CTRL, 0x0a), }; +static const struct qmp_phy_init_tbl sm8475_ufsphy_serdes[] = { + QMP_PHY_INIT_CFG(QSERDES_V6_COM_SYSCLK_EN_SEL, 0xd9), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_CMN_CONFIG_1, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_HSCLK_SEL_1, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_HSCLK_HS_SWITCH_SEL_1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP_EN, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_VCO_TUNE_INITVAL2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_DEC_START_MODE0, 0x82), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_RCTRL_MODE0, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_CCTRL_MODE0, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP1_MODE0, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP2_MODE0, 0x0c), +}; + +static const struct qmp_phy_init_tbl sm8475_ufsphy_g4_serdes[] = { + QMP_PHY_INIT_CFG(QSERDES_V6_COM_VCO_TUNE_MAP, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_IVCO, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_CP_CTRL_MODE0, 0x14), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_DEC_START_MODE1, 0x98), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_CP_CTRL_MODE1, 0x14), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_RCTRL_MODE1, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_CCTRL_MODE1, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP1_MODE1, 0x32), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP2_MODE1, 0x0f), +}; + +static const struct qmp_phy_init_tbl sm8475_ufsphy_g4_pcs[] = { + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x0b), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x04), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x04), +}; + static const struct qmp_phy_init_tbl sm8550_ufsphy_serdes[] = { QMP_PHY_INIT_CFG(QSERDES_V6_COM_SYSCLK_EN_SEL, 0xd9), QMP_PHY_INIT_CFG(QSERDES_V6_COM_CMN_CONFIG_1, 0x16), @@ -830,17 +862,20 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_serdes[] = { QMP_PHY_INIT_CFG(QSERDES_V6_COM_HSCLK_SEL_1, 0x11), QMP_PHY_INIT_CFG(QSERDES_V6_COM_HSCLK_HS_SWITCH_SEL_1, 0x00), QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP_EN, 0x01), - QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_IVCO, 0x0f), - QMP_PHY_INIT_CFG(QSERDES_V6_COM_VCO_TUNE_MAP, 0x44), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_IVCO, 0x1f), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_IVCO_MODE1, 0x1f), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_CMN_IETRIM, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_CMN_IPTRIM, 0x17), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_VCO_TUNE_MAP, 0x04), QMP_PHY_INIT_CFG(QSERDES_V6_COM_VCO_TUNE_INITVAL2, 0x00), QMP_PHY_INIT_CFG(QSERDES_V6_COM_DEC_START_MODE0, 0x41), - QMP_PHY_INIT_CFG(QSERDES_V6_COM_CP_CTRL_MODE0, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_CP_CTRL_MODE0, 0x06), QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_RCTRL_MODE0, 0x18), QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_CCTRL_MODE0, 0x14), QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP1_MODE0, 0x7f), QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP2_MODE0, 0x06), QMP_PHY_INIT_CFG(QSERDES_V6_COM_DEC_START_MODE1, 0x4c), - QMP_PHY_INIT_CFG(QSERDES_V6_COM_CP_CTRL_MODE1, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V6_COM_CP_CTRL_MODE1, 0x06), QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_RCTRL_MODE1, 0x18), QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_CCTRL_MODE1, 0x14), QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP1_MODE1, 0x99), @@ -848,17 +883,28 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_serdes[] = { }; static const struct qmp_phy_init_tbl sm8650_ufsphy_tx[] = { - QMP_PHY_INIT_CFG(QSERDES_UFS_V6_TX_LANE_MODE_1, 0x05), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_TX_LANE_MODE_1, 0x01), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_TX_RES_CODE_LANE_OFFSET_TX, 0x07), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_TX_RES_CODE_LANE_OFFSET_RX, 0x0e), }; static const struct qmp_phy_init_tbl sm8650_ufsphy_rx[] = { QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_FO_GAIN_RATE2, 0x0c), - QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_FO_GAIN_RATE4, 0x0f), - QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_VGA_CAL_MAN_VAL, 0x0e), - QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B0, 0xc2), - QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B1, 0xc2), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_FO_GAIN_RATE4, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_SO_GAIN_RATE4, 0x04), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x14), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_PI_CONTROLS, 0x07), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_OFFSET_ADAPTOR_CNTRL3, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_FASTLOCK_COUNT_HIGH_RATE4, 0x02), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_FASTLOCK_FO_GAIN_RATE4, 0x1c), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_FASTLOCK_SO_GAIN_RATE4, 0x06), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_VGA_CAL_MAN_VAL, 0x3e), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_RX_EQU_ADAPTOR_CNTRL4, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B0, 0xce), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B1, 0xce), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B2, 0x18), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B3, 0x1a), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B4, 0x0f), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B6, 0x60), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE2_B3, 0x9e), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE2_B6, 0x60), @@ -866,23 +912,41 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_rx[] = { QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE3_B4, 0x0e), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE3_B5, 0x36), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE3_B8, 0x02), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE4_B0, 0x24), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE4_B1, 0x24), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE4_B2, 0x20), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE4_B3, 0xb9), - QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE4_B6, 0xff), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE4_B4, 0x4f), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_SO_SATURATION, 0x1f), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_PI_CTRL1, 0x94), QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_RX_TERM_BW_CTRL0, 0xfa), + QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_DLL0_FTUNE_CTRL, 0x30), }; static const struct qmp_phy_init_tbl sm8650_ufsphy_pcs[] = { - QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x00), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x02), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_MID_TERM_CTRL1, 0x43), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PCS_CTRL1, 0xc1), - QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0f), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_SIGDET_CTRL2, 0x68), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S4, 0x0e), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S5, 0x12), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S6, 0x15), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S7, 0x19), +}; + +static const struct qmp_phy_init_tbl sm8650_ufsphy_g4_pcs[] = { + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x13), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x04), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x04), - QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0f), - QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_SIGDET_CTRL2, 0x69), - QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x02), +}; + +static const struct qmp_phy_init_tbl sm8650_ufsphy_g5_pcs[] = { + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x05), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x05), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HS_G5_SYNC_LENGTH_CAPABILITY, 0x4d), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSG5_SYNC_WAIT_TIME, 0x9e), }; struct qmp_ufs_offsets { @@ -1346,6 +1410,42 @@ static const struct qmp_phy_cfg sm8450_ufsphy_cfg = { .regs = ufsphy_v5_regs_layout, }; +static const struct qmp_phy_cfg sm8475_ufsphy_cfg = { + .lanes = 2, + + .offsets = &qmp_ufs_offsets_v6, + .max_supported_gear = UFS_HS_G4, + + .tbls = { + .serdes = sm8475_ufsphy_serdes, + .serdes_num = ARRAY_SIZE(sm8475_ufsphy_serdes), + .tx = sm8550_ufsphy_tx, + .tx_num = ARRAY_SIZE(sm8550_ufsphy_tx), + .rx = sm8550_ufsphy_rx, + .rx_num = ARRAY_SIZE(sm8550_ufsphy_rx), + .pcs = sm8550_ufsphy_pcs, + .pcs_num = ARRAY_SIZE(sm8550_ufsphy_pcs), + }, + .tbls_hs_b = { + .serdes = sm8550_ufsphy_hs_b_serdes, + .serdes_num = ARRAY_SIZE(sm8550_ufsphy_hs_b_serdes), + }, + .tbls_hs_overlay[0] = { + .serdes = sm8475_ufsphy_g4_serdes, + .serdes_num = ARRAY_SIZE(sm8475_ufsphy_g4_serdes), + .tx = sm8550_ufsphy_g4_tx, + .tx_num = ARRAY_SIZE(sm8550_ufsphy_g4_tx), + .rx = sm8550_ufsphy_g4_rx, + .rx_num = ARRAY_SIZE(sm8550_ufsphy_g4_rx), + .pcs = sm8475_ufsphy_g4_pcs, + .pcs_num = ARRAY_SIZE(sm8475_ufsphy_g4_pcs), + .max_gear = UFS_HS_G4, + }, + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = ufsphy_v6_regs_layout, +}; + static const struct qmp_phy_cfg sm8550_ufsphy_cfg = { .lanes = 2, @@ -1407,6 +1507,17 @@ static const struct qmp_phy_cfg sm8650_ufsphy_cfg = { .pcs = sm8650_ufsphy_pcs, .pcs_num = ARRAY_SIZE(sm8650_ufsphy_pcs), }, + .tbls_hs_overlay[0] = { + .pcs = sm8650_ufsphy_g4_pcs, + .pcs_num = ARRAY_SIZE(sm8650_ufsphy_g4_pcs), + .max_gear = UFS_HS_G4, + }, + .tbls_hs_overlay[1] = { + .pcs = sm8650_ufsphy_g5_pcs, + .pcs_num = ARRAY_SIZE(sm8650_ufsphy_g5_pcs), + .max_gear = UFS_HS_G5, + }, + .vreg_list = qmp_phy_vreg_l, .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), .regs = ufsphy_v6_regs_layout, @@ -1942,6 +2053,9 @@ static const struct of_device_id qmp_ufs_of_match_table[] = { .compatible = "qcom,sm8450-qmp-ufs-phy", .data = &sm8450_ufsphy_cfg, }, { + .compatible = "qcom,sm8475-qmp-ufs-phy", + .data = &sm8475_ufsphy_cfg, + }, { .compatible = "qcom,sm8550-qmp-ufs-phy", .data = &sm8550_ufsphy_cfg, }, { diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c index 85253936fac3..c174463c58a3 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c @@ -337,6 +337,29 @@ static const struct qmp_phy_init_tbl msm8996_usb3_pcs_tbl[] = { QMP_PHY_INIT_CFG(QPHY_V2_PCS_POWER_STATE_CONFIG2, 0x08), }; +static const struct qmp_phy_init_tbl qdu1000_usb3_uniphy_pcs_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG1, 0xc4), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG2, 0x89), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG3, 0x20), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG6, 0x13), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_L, 0xe7), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_H, 0x03), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RX_SIGDET_LVL, 0xaa), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_PCS_TX_RX_CONFIG, 0x0c), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_CDR_RESET_TIME, 0x0a), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG1, 0x88), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG2, 0x13), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG1, 0x4b), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG5, 0x10), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_REFGEN_REQ_CONFIG1, 0x21), +}; + +static const struct qmp_phy_init_tbl qdu1000_usb3_uniphy_pcs_usb_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V4_PCS_USB3_RXEQTRAINING_DFE_TIME_S2, 0x07), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_USB3_LFPS_DET_HIGH_COUNT_VAL, 0xf8), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_USB3_POWER_STATE_CONFIG1, 0x6f), +}; + static const struct qmp_phy_init_tbl qmp_v3_usb3_uniphy_serdes_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V3_COM_PLL_IVCO, 0x07), QMP_PHY_INIT_CFG(QSERDES_V3_COM_SYSCLK_EN_SEL, 0x14), @@ -1400,6 +1423,27 @@ static const struct qmp_phy_cfg msm8996_usb3phy_cfg = { .regs = qmp_v2_usb3phy_regs_layout, }; +static const struct qmp_phy_cfg qdu1000_usb3_uniphy_cfg = { + .offsets = &qmp_usb_offsets_v5, + + .serdes_tbl = sm8150_usb3_uniphy_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_uniphy_serdes_tbl), + .tx_tbl = sm8350_usb3_uniphy_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sm8350_usb3_uniphy_tx_tbl), + .rx_tbl = sm8350_usb3_uniphy_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sm8350_usb3_uniphy_rx_tbl), + .pcs_tbl = qdu1000_usb3_uniphy_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(qdu1000_usb3_uniphy_pcs_tbl), + .pcs_usb_tbl = qdu1000_usb3_uniphy_pcs_usb_tbl, + .pcs_usb_tbl_num = ARRAY_SIZE(qdu1000_usb3_uniphy_pcs_usb_tbl), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = qmp_v4_usb3phy_regs_layout, + .pcs_usb_offset = 0x1000, + + .has_pwrdn_delay = true, +}; + static const struct qmp_phy_cfg sa8775p_usb3_uniphy_cfg = { .offsets = &qmp_usb_offsets_v5, @@ -2203,6 +2247,9 @@ static const struct of_device_id qmp_usb_of_match_table[] = { .compatible = "qcom,msm8996-qmp-usb3-phy", .data = &msm8996_usb3phy_cfg, }, { + .compatible = "qcom,qdu1000-qmp-usb3-uni-phy", + .data = &qdu1000_usb3_uniphy_cfg, + }, { .compatible = "qcom,sa8775p-qmp-usb3-uni-phy", .data = &sa8775p_usb3_uniphy_cfg, }, { diff --git a/drivers/phy/rockchip/Kconfig b/drivers/phy/rockchip/Kconfig index b60a4b60451e..08b0f4345760 100644 --- a/drivers/phy/rockchip/Kconfig +++ b/drivers/phy/rockchip/Kconfig @@ -116,3 +116,15 @@ config PHY_ROCKCHIP_USB select GENERIC_PHY help Enable this to support the Rockchip USB 2.0 PHY. + +config PHY_ROCKCHIP_USBDP + tristate "Rockchip USBDP COMBO PHY Driver" + depends on ARCH_ROCKCHIP && OF + depends on TYPEC + select GENERIC_PHY + help + Enable this to support the Rockchip USB3.0/DP combo PHY with + Samsung IP block. This is required for USB3 support on RK3588. + + To compile this driver as a module, choose M here: the module + will be called phy-rockchip-usbdp diff --git a/drivers/phy/rockchip/Makefile b/drivers/phy/rockchip/Makefile index 3d911304e654..010a824e32ce 100644 --- a/drivers/phy/rockchip/Makefile +++ b/drivers/phy/rockchip/Makefile @@ -12,3 +12,4 @@ obj-$(CONFIG_PHY_ROCKCHIP_SAMSUNG_HDPTX) += phy-rockchip-samsung-hdptx.o obj-$(CONFIG_PHY_ROCKCHIP_SNPS_PCIE3) += phy-rockchip-snps-pcie3.o obj-$(CONFIG_PHY_ROCKCHIP_TYPEC) += phy-rockchip-typec.o obj-$(CONFIG_PHY_ROCKCHIP_USB) += phy-rockchip-usb.o +obj-$(CONFIG_PHY_ROCKCHIP_USBDP) += phy-rockchip-usbdp.o diff --git a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c index bf74e429ff46..0a9989e41237 100644 --- a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c +++ b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c @@ -248,7 +248,7 @@ static int rockchip_combphy_exit(struct phy *phy) return 0; } -static const struct phy_ops rochchip_combphy_ops = { +static const struct phy_ops rockchip_combphy_ops = { .init = rockchip_combphy_init, .exit = rockchip_combphy_exit, .owner = THIS_MODULE, @@ -364,7 +364,7 @@ static int rockchip_combphy_probe(struct platform_device *pdev) return ret; } - priv->phy = devm_phy_create(dev, NULL, &rochchip_combphy_ops); + priv->phy = devm_phy_create(dev, NULL, &rockchip_combphy_ops); if (IS_ERR(priv->phy)) { dev_err(dev, "failed to create combphy\n"); return PTR_ERR(priv->phy); diff --git a/drivers/phy/rockchip/phy-rockchip-snps-pcie3.c b/drivers/phy/rockchip/phy-rockchip-snps-pcie3.c index 9857ee45b89e..4e8ffd173096 100644 --- a/drivers/phy/rockchip/phy-rockchip-snps-pcie3.c +++ b/drivers/phy/rockchip/phy-rockchip-snps-pcie3.c @@ -35,11 +35,17 @@ #define RK3588_PCIE3PHY_GRF_CMN_CON0 0x0 #define RK3588_PCIE3PHY_GRF_PHY0_STATUS1 0x904 #define RK3588_PCIE3PHY_GRF_PHY1_STATUS1 0xa04 +#define RK3588_PCIE3PHY_GRF_PHY0_LN0_CON1 0x1004 +#define RK3588_PCIE3PHY_GRF_PHY0_LN1_CON1 0x1104 +#define RK3588_PCIE3PHY_GRF_PHY1_LN0_CON1 0x2004 +#define RK3588_PCIE3PHY_GRF_PHY1_LN1_CON1 0x2104 #define RK3588_SRAM_INIT_DONE(reg) (reg & BIT(0)) #define RK3588_BIFURCATION_LANE_0_1 BIT(0) #define RK3588_BIFURCATION_LANE_2_3 BIT(1) #define RK3588_LANE_AGGREGATION BIT(2) +#define RK3588_RX_CMN_REFCLK_MODE_EN ((BIT(7) << 16) | BIT(7)) +#define RK3588_RX_CMN_REFCLK_MODE_DIS (BIT(7) << 16) #define RK3588_PCIE1LN_SEL_EN (GENMASK(1, 0) << 16) #define RK3588_PCIE30_PHY_MODE_EN (GENMASK(2, 0) << 16) @@ -60,6 +66,7 @@ struct rockchip_p3phy_priv { int num_clks; int num_lanes; u32 lanes[4]; + u32 rx_cmn_refclk_mode[4]; }; struct rockchip_p3phy_ops { @@ -137,6 +144,19 @@ static int rockchip_p3phy_rk3588_init(struct rockchip_p3phy_priv *priv) u8 mode = RK3588_LANE_AGGREGATION; /* default */ int ret; + regmap_write(priv->phy_grf, RK3588_PCIE3PHY_GRF_PHY0_LN0_CON1, + priv->rx_cmn_refclk_mode[0] ? RK3588_RX_CMN_REFCLK_MODE_EN : + RK3588_RX_CMN_REFCLK_MODE_DIS); + regmap_write(priv->phy_grf, RK3588_PCIE3PHY_GRF_PHY0_LN1_CON1, + priv->rx_cmn_refclk_mode[1] ? RK3588_RX_CMN_REFCLK_MODE_EN : + RK3588_RX_CMN_REFCLK_MODE_DIS); + regmap_write(priv->phy_grf, RK3588_PCIE3PHY_GRF_PHY1_LN0_CON1, + priv->rx_cmn_refclk_mode[2] ? RK3588_RX_CMN_REFCLK_MODE_EN : + RK3588_RX_CMN_REFCLK_MODE_DIS); + regmap_write(priv->phy_grf, RK3588_PCIE3PHY_GRF_PHY1_LN1_CON1, + priv->rx_cmn_refclk_mode[3] ? RK3588_RX_CMN_REFCLK_MODE_EN : + RK3588_RX_CMN_REFCLK_MODE_DIS); + /* Deassert PCIe PMA output clamp mode */ regmap_write(priv->phy_grf, RK3588_PCIE3PHY_GRF_CMN_CON0, BIT(8) | BIT(24)); @@ -182,7 +202,7 @@ static const struct rockchip_p3phy_ops rk3588_ops = { .phy_init = rockchip_p3phy_rk3588_init, }; -static int rochchip_p3phy_init(struct phy *phy) +static int rockchip_p3phy_init(struct phy *phy) { struct rockchip_p3phy_priv *priv = phy_get_drvdata(phy); int ret; @@ -205,7 +225,7 @@ static int rochchip_p3phy_init(struct phy *phy) return ret; } -static int rochchip_p3phy_exit(struct phy *phy) +static int rockchip_p3phy_exit(struct phy *phy) { struct rockchip_p3phy_priv *priv = phy_get_drvdata(phy); @@ -214,9 +234,9 @@ static int rochchip_p3phy_exit(struct phy *phy) return 0; } -static const struct phy_ops rochchip_p3phy_ops = { - .init = rochchip_p3phy_init, - .exit = rochchip_p3phy_exit, +static const struct phy_ops rockchip_p3phy_ops = { + .init = rockchip_p3phy_init, + .exit = rockchip_p3phy_exit, .set_mode = rockchip_p3phy_set_mode, .owner = THIS_MODULE, }; @@ -275,7 +295,24 @@ static int rockchip_p3phy_probe(struct platform_device *pdev) return priv->num_lanes; } - priv->phy = devm_phy_create(dev, NULL, &rochchip_p3phy_ops); + ret = of_property_read_variable_u32_array(dev->of_node, + "rockchip,rx-common-refclk-mode", + priv->rx_cmn_refclk_mode, 1, + ARRAY_SIZE(priv->rx_cmn_refclk_mode)); + /* + * if no rockchip,rx-common-refclk-mode, assume enabled for all lanes in + * order to be DT backwards compatible. (Since HW reset val is enabled.) + */ + if (ret == -EINVAL) { + for (int i = 0; i < ARRAY_SIZE(priv->rx_cmn_refclk_mode); i++) + priv->rx_cmn_refclk_mode[i] = 1; + } else if (ret < 0) { + dev_err(dev, "failed to read rockchip,rx-common-refclk-mode property %d\n", + ret); + return ret; + } + + priv->phy = devm_phy_create(dev, NULL, &rockchip_p3phy_ops); if (IS_ERR(priv->phy)) { dev_err(dev, "failed to create combphy\n"); return PTR_ERR(priv->phy); diff --git a/drivers/phy/rockchip/phy-rockchip-usbdp.c b/drivers/phy/rockchip/phy-rockchip-usbdp.c new file mode 100644 index 000000000000..2c51e5c62d3e --- /dev/null +++ b/drivers/phy/rockchip/phy-rockchip-usbdp.c @@ -0,0 +1,1608 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Rockchip USBDP Combo PHY with Samsung IP block driver + * + * Copyright (C) 2021-2024 Rockchip Electronics Co., Ltd + * Copyright (C) 2024 Collabora Ltd + */ + +#include <dt-bindings/phy/phy.h> +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio.h> +#include <linux/mfd/syscon.h> +#include <linux/mod_devicetable.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/property.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include <linux/usb/ch9.h> +#include <linux/usb/typec_dp.h> +#include <linux/usb/typec_mux.h> + +/* USBDP PHY Register Definitions */ +#define UDPHY_PCS 0x4000 +#define UDPHY_PMA 0x8000 + +/* VO0 GRF Registers */ +#define DP_SINK_HPD_CFG BIT(11) +#define DP_SINK_HPD_SEL BIT(10) +#define DP_AUX_DIN_SEL BIT(9) +#define DP_AUX_DOUT_SEL BIT(8) +#define DP_LANE_SEL_N(n) GENMASK(2 * (n) + 1, 2 * (n)) +#define DP_LANE_SEL_ALL GENMASK(7, 0) + +/* PMA CMN Registers */ +#define CMN_LANE_MUX_AND_EN_OFFSET 0x0288 /* cmn_reg00A2 */ +#define CMN_DP_LANE_MUX_N(n) BIT((n) + 4) +#define CMN_DP_LANE_EN_N(n) BIT(n) +#define CMN_DP_LANE_MUX_ALL GENMASK(7, 4) +#define CMN_DP_LANE_EN_ALL GENMASK(3, 0) + +#define CMN_DP_LINK_OFFSET 0x28c /* cmn_reg00A3 */ +#define CMN_DP_TX_LINK_BW GENMASK(6, 5) +#define CMN_DP_TX_LANE_SWAP_EN BIT(2) + +#define CMN_SSC_EN_OFFSET 0x2d0 /* cmn_reg00B4 */ +#define CMN_ROPLL_SSC_EN BIT(1) +#define CMN_LCPLL_SSC_EN BIT(0) + +#define CMN_ANA_LCPLL_DONE_OFFSET 0x0350 /* cmn_reg00D4 */ +#define CMN_ANA_LCPLL_LOCK_DONE BIT(7) +#define CMN_ANA_LCPLL_AFC_DONE BIT(6) + +#define CMN_ANA_ROPLL_DONE_OFFSET 0x0354 /* cmn_reg00D5 */ +#define CMN_ANA_ROPLL_LOCK_DONE BIT(1) +#define CMN_ANA_ROPLL_AFC_DONE BIT(0) + +#define CMN_DP_RSTN_OFFSET 0x038c /* cmn_reg00E3 */ +#define CMN_DP_INIT_RSTN BIT(3) +#define CMN_DP_CMN_RSTN BIT(2) +#define CMN_CDR_WTCHDG_EN BIT(1) +#define CMN_CDR_WTCHDG_MSK_CDR_EN BIT(0) + +#define TRSV_ANA_TX_CLK_OFFSET_N(n) (0x854 + (n) * 0x800) /* trsv_reg0215 */ +#define LN_ANA_TX_SER_TXCLK_INV BIT(1) + +#define TRSV_LN0_MON_RX_CDR_DONE_OFFSET 0x0b84 /* trsv_reg02E1 */ +#define TRSV_LN0_MON_RX_CDR_LOCK_DONE BIT(0) + +#define TRSV_LN2_MON_RX_CDR_DONE_OFFSET 0x1b84 /* trsv_reg06E1 */ +#define TRSV_LN2_MON_RX_CDR_LOCK_DONE BIT(0) + +#define BIT_WRITEABLE_SHIFT 16 +#define PHY_AUX_DP_DATA_POL_NORMAL 0 +#define PHY_AUX_DP_DATA_POL_INVERT 1 +#define PHY_LANE_MUX_USB 0 +#define PHY_LANE_MUX_DP 1 + +enum { + DP_BW_RBR, + DP_BW_HBR, + DP_BW_HBR2, + DP_BW_HBR3, +}; + +enum { + UDPHY_MODE_NONE = 0, + UDPHY_MODE_USB = BIT(0), + UDPHY_MODE_DP = BIT(1), + UDPHY_MODE_DP_USB = BIT(1) | BIT(0), +}; + +struct rk_udphy_grf_reg { + unsigned int offset; + unsigned int disable; + unsigned int enable; +}; + +#define _RK_UDPHY_GEN_GRF_REG(offset, mask, disable, enable) \ +{\ + offset, \ + FIELD_PREP_CONST(mask, disable) | (mask << BIT_WRITEABLE_SHIFT), \ + FIELD_PREP_CONST(mask, enable) | (mask << BIT_WRITEABLE_SHIFT), \ +} + +#define RK_UDPHY_GEN_GRF_REG(offset, bitend, bitstart, disable, enable) \ + _RK_UDPHY_GEN_GRF_REG(offset, GENMASK(bitend, bitstart), disable, enable) + +struct rk_udphy_grf_cfg { + /* u2phy-grf */ + struct rk_udphy_grf_reg bvalid_phy_con; + struct rk_udphy_grf_reg bvalid_grf_con; + + /* usb-grf */ + struct rk_udphy_grf_reg usb3otg0_cfg; + struct rk_udphy_grf_reg usb3otg1_cfg; + + /* usbdpphy-grf */ + struct rk_udphy_grf_reg low_pwrn; + struct rk_udphy_grf_reg rx_lfps; +}; + +struct rk_udphy_vogrf_cfg { + /* vo-grf */ + struct rk_udphy_grf_reg hpd_trigger; + u32 dp_lane_reg; +}; + +struct rk_udphy_dp_tx_drv_ctrl { + u32 trsv_reg0204; + u32 trsv_reg0205; + u32 trsv_reg0206; + u32 trsv_reg0207; +}; + +struct rk_udphy_cfg { + unsigned int num_phys; + unsigned int phy_ids[2]; + /* resets to be requested */ + const char * const *rst_list; + int num_rsts; + + struct rk_udphy_grf_cfg grfcfg; + struct rk_udphy_vogrf_cfg vogrfcfg[2]; + const struct rk_udphy_dp_tx_drv_ctrl (*dp_tx_ctrl_cfg[4])[4]; + const struct rk_udphy_dp_tx_drv_ctrl (*dp_tx_ctrl_cfg_typec[4])[4]; +}; + +struct rk_udphy { + struct device *dev; + struct regmap *pma_regmap; + struct regmap *u2phygrf; + struct regmap *udphygrf; + struct regmap *usbgrf; + struct regmap *vogrf; + struct typec_switch_dev *sw; + struct typec_mux_dev *mux; + struct mutex mutex; /* mutex to protect access to individual PHYs */ + + /* clocks and rests */ + int num_clks; + struct clk_bulk_data *clks; + struct clk *refclk; + int num_rsts; + struct reset_control_bulk_data *rsts; + + /* PHY status management */ + bool flip; + bool mode_change; + u8 mode; + u8 status; + + /* utilized for USB */ + bool hs; /* flag for high-speed */ + + /* utilized for DP */ + struct gpio_desc *sbu1_dc_gpio; + struct gpio_desc *sbu2_dc_gpio; + u32 lane_mux_sel[4]; + u32 dp_lane_sel[4]; + u32 dp_aux_dout_sel; + u32 dp_aux_din_sel; + bool dp_sink_hpd_sel; + bool dp_sink_hpd_cfg; + u8 bw; + int id; + + bool dp_in_use; + + /* PHY const config */ + const struct rk_udphy_cfg *cfgs; + + /* PHY devices */ + struct phy *phy_dp; + struct phy *phy_u3; +}; + +static const struct rk_udphy_dp_tx_drv_ctrl rk3588_dp_tx_drv_ctrl_rbr_hbr[4][4] = { + /* voltage swing 0, pre-emphasis 0->3 */ + { + { 0x20, 0x10, 0x42, 0xe5 }, + { 0x26, 0x14, 0x42, 0xe5 }, + { 0x29, 0x18, 0x42, 0xe5 }, + { 0x2b, 0x1c, 0x43, 0xe7 }, + }, + + /* voltage swing 1, pre-emphasis 0->2 */ + { + { 0x23, 0x10, 0x42, 0xe7 }, + { 0x2a, 0x17, 0x43, 0xe7 }, + { 0x2b, 0x1a, 0x43, 0xe7 }, + }, + + /* voltage swing 2, pre-emphasis 0->1 */ + { + { 0x27, 0x10, 0x42, 0xe7 }, + { 0x2b, 0x17, 0x43, 0xe7 }, + }, + + /* voltage swing 3, pre-emphasis 0 */ + { + { 0x29, 0x10, 0x43, 0xe7 }, + }, +}; + +static const struct rk_udphy_dp_tx_drv_ctrl rk3588_dp_tx_drv_ctrl_rbr_hbr_typec[4][4] = { + /* voltage swing 0, pre-emphasis 0->3 */ + { + { 0x20, 0x10, 0x42, 0xe5 }, + { 0x26, 0x14, 0x42, 0xe5 }, + { 0x29, 0x18, 0x42, 0xe5 }, + { 0x2b, 0x1c, 0x43, 0xe7 }, + }, + + /* voltage swing 1, pre-emphasis 0->2 */ + { + { 0x23, 0x10, 0x42, 0xe7 }, + { 0x2a, 0x17, 0x43, 0xe7 }, + { 0x2b, 0x1a, 0x43, 0xe7 }, + }, + + /* voltage swing 2, pre-emphasis 0->1 */ + { + { 0x27, 0x10, 0x43, 0x67 }, + { 0x2b, 0x17, 0x43, 0xe7 }, + }, + + /* voltage swing 3, pre-emphasis 0 */ + { + { 0x29, 0x10, 0x43, 0xe7 }, + }, +}; + +static const struct rk_udphy_dp_tx_drv_ctrl rk3588_dp_tx_drv_ctrl_hbr2[4][4] = { + /* voltage swing 0, pre-emphasis 0->3 */ + { + { 0x21, 0x10, 0x42, 0xe5 }, + { 0x26, 0x14, 0x42, 0xe5 }, + { 0x26, 0x16, 0x43, 0xe5 }, + { 0x2a, 0x19, 0x43, 0xe7 }, + }, + + /* voltage swing 1, pre-emphasis 0->2 */ + { + { 0x24, 0x10, 0x42, 0xe7 }, + { 0x2a, 0x17, 0x43, 0xe7 }, + { 0x2b, 0x1a, 0x43, 0xe7 }, + }, + + /* voltage swing 2, pre-emphasis 0->1 */ + { + { 0x28, 0x10, 0x42, 0xe7 }, + { 0x2b, 0x17, 0x43, 0xe7 }, + }, + + /* voltage swing 3, pre-emphasis 0 */ + { + { 0x28, 0x10, 0x43, 0xe7 }, + }, +}; + +static const struct rk_udphy_dp_tx_drv_ctrl rk3588_dp_tx_drv_ctrl_hbr3[4][4] = { + /* voltage swing 0, pre-emphasis 0->3 */ + { + { 0x21, 0x10, 0x42, 0xe5 }, + { 0x26, 0x14, 0x42, 0xe5 }, + { 0x26, 0x16, 0x43, 0xe5 }, + { 0x29, 0x18, 0x43, 0xe7 }, + }, + + /* voltage swing 1, pre-emphasis 0->2 */ + { + { 0x24, 0x10, 0x42, 0xe7 }, + { 0x2a, 0x18, 0x43, 0xe7 }, + { 0x2b, 0x1b, 0x43, 0xe7 } + }, + + /* voltage swing 2, pre-emphasis 0->1 */ + { + { 0x27, 0x10, 0x42, 0xe7 }, + { 0x2b, 0x18, 0x43, 0xe7 } + }, + + /* voltage swing 3, pre-emphasis 0 */ + { + { 0x28, 0x10, 0x43, 0xe7 }, + }, +}; + +static const struct reg_sequence rk_udphy_24m_refclk_cfg[] = { + {0x0090, 0x68}, {0x0094, 0x68}, + {0x0128, 0x24}, {0x012c, 0x44}, + {0x0130, 0x3f}, {0x0134, 0x44}, + {0x015c, 0xa9}, {0x0160, 0x71}, + {0x0164, 0x71}, {0x0168, 0xa9}, + {0x0174, 0xa9}, {0x0178, 0x71}, + {0x017c, 0x71}, {0x0180, 0xa9}, + {0x018c, 0x41}, {0x0190, 0x00}, + {0x0194, 0x05}, {0x01ac, 0x2a}, + {0x01b0, 0x17}, {0x01b4, 0x17}, + {0x01b8, 0x2a}, {0x01c8, 0x04}, + {0x01cc, 0x08}, {0x01d0, 0x08}, + {0x01d4, 0x04}, {0x01d8, 0x20}, + {0x01dc, 0x01}, {0x01e0, 0x09}, + {0x01e4, 0x03}, {0x01f0, 0x29}, + {0x01f4, 0x02}, {0x01f8, 0x02}, + {0x01fc, 0x29}, {0x0208, 0x2a}, + {0x020c, 0x17}, {0x0210, 0x17}, + {0x0214, 0x2a}, {0x0224, 0x20}, + {0x03f0, 0x0a}, {0x03f4, 0x07}, + {0x03f8, 0x07}, {0x03fc, 0x0c}, + {0x0404, 0x12}, {0x0408, 0x1a}, + {0x040c, 0x1a}, {0x0410, 0x3f}, + {0x0ce0, 0x68}, {0x0ce8, 0xd0}, + {0x0cf0, 0x87}, {0x0cf8, 0x70}, + {0x0d00, 0x70}, {0x0d08, 0xa9}, + {0x1ce0, 0x68}, {0x1ce8, 0xd0}, + {0x1cf0, 0x87}, {0x1cf8, 0x70}, + {0x1d00, 0x70}, {0x1d08, 0xa9}, + {0x0a3c, 0xd0}, {0x0a44, 0xd0}, + {0x0a48, 0x01}, {0x0a4c, 0x0d}, + {0x0a54, 0xe0}, {0x0a5c, 0xe0}, + {0x0a64, 0xa8}, {0x1a3c, 0xd0}, + {0x1a44, 0xd0}, {0x1a48, 0x01}, + {0x1a4c, 0x0d}, {0x1a54, 0xe0}, + {0x1a5c, 0xe0}, {0x1a64, 0xa8} +}; + +static const struct reg_sequence rk_udphy_26m_refclk_cfg[] = { + {0x0830, 0x07}, {0x085c, 0x80}, + {0x1030, 0x07}, {0x105c, 0x80}, + {0x1830, 0x07}, {0x185c, 0x80}, + {0x2030, 0x07}, {0x205c, 0x80}, + {0x0228, 0x38}, {0x0104, 0x44}, + {0x0248, 0x44}, {0x038c, 0x02}, + {0x0878, 0x04}, {0x1878, 0x04}, + {0x0898, 0x77}, {0x1898, 0x77}, + {0x0054, 0x01}, {0x00e0, 0x38}, + {0x0060, 0x24}, {0x0064, 0x77}, + {0x0070, 0x76}, {0x0234, 0xe8}, + {0x0af4, 0x15}, {0x1af4, 0x15}, + {0x081c, 0xe5}, {0x181c, 0xe5}, + {0x099c, 0x48}, {0x199c, 0x48}, + {0x09a4, 0x07}, {0x09a8, 0x22}, + {0x19a4, 0x07}, {0x19a8, 0x22}, + {0x09b8, 0x3e}, {0x19b8, 0x3e}, + {0x09e4, 0x02}, {0x19e4, 0x02}, + {0x0a34, 0x1e}, {0x1a34, 0x1e}, + {0x0a98, 0x2f}, {0x1a98, 0x2f}, + {0x0c30, 0x0e}, {0x0c48, 0x06}, + {0x1c30, 0x0e}, {0x1c48, 0x06}, + {0x028c, 0x18}, {0x0af0, 0x00}, + {0x1af0, 0x00} +}; + +static const struct reg_sequence rk_udphy_init_sequence[] = { + {0x0104, 0x44}, {0x0234, 0xe8}, + {0x0248, 0x44}, {0x028c, 0x18}, + {0x081c, 0xe5}, {0x0878, 0x00}, + {0x0994, 0x1c}, {0x0af0, 0x00}, + {0x181c, 0xe5}, {0x1878, 0x00}, + {0x1994, 0x1c}, {0x1af0, 0x00}, + {0x0428, 0x60}, {0x0d58, 0x33}, + {0x1d58, 0x33}, {0x0990, 0x74}, + {0x0d64, 0x17}, {0x08c8, 0x13}, + {0x1990, 0x74}, {0x1d64, 0x17}, + {0x18c8, 0x13}, {0x0d90, 0x40}, + {0x0da8, 0x40}, {0x0dc0, 0x40}, + {0x0dd8, 0x40}, {0x1d90, 0x40}, + {0x1da8, 0x40}, {0x1dc0, 0x40}, + {0x1dd8, 0x40}, {0x03c0, 0x30}, + {0x03c4, 0x06}, {0x0e10, 0x00}, + {0x1e10, 0x00}, {0x043c, 0x0f}, + {0x0d2c, 0xff}, {0x1d2c, 0xff}, + {0x0d34, 0x0f}, {0x1d34, 0x0f}, + {0x08fc, 0x2a}, {0x0914, 0x28}, + {0x0a30, 0x03}, {0x0e38, 0x03}, + {0x0ecc, 0x27}, {0x0ed0, 0x22}, + {0x0ed4, 0x26}, {0x18fc, 0x2a}, + {0x1914, 0x28}, {0x1a30, 0x03}, + {0x1e38, 0x03}, {0x1ecc, 0x27}, + {0x1ed0, 0x22}, {0x1ed4, 0x26}, + {0x0048, 0x0f}, {0x0060, 0x3c}, + {0x0064, 0xf7}, {0x006c, 0x20}, + {0x0070, 0x7d}, {0x0074, 0x68}, + {0x0af4, 0x1a}, {0x1af4, 0x1a}, + {0x0440, 0x3f}, {0x10d4, 0x08}, + {0x20d4, 0x08}, {0x00d4, 0x30}, + {0x0024, 0x6e}, +}; + +static inline int rk_udphy_grfreg_write(struct regmap *base, + const struct rk_udphy_grf_reg *reg, bool en) +{ + return regmap_write(base, reg->offset, en ? reg->enable : reg->disable); +} + +static int rk_udphy_clk_init(struct rk_udphy *udphy, struct device *dev) +{ + int i; + + udphy->num_clks = devm_clk_bulk_get_all(dev, &udphy->clks); + if (udphy->num_clks < 1) + return -ENODEV; + + /* used for configure phy reference clock frequency */ + for (i = 0; i < udphy->num_clks; i++) { + if (!strncmp(udphy->clks[i].id, "refclk", 6)) { + udphy->refclk = udphy->clks[i].clk; + break; + } + } + + if (!udphy->refclk) + return dev_err_probe(udphy->dev, -EINVAL, "no refclk found\n"); + + return 0; +} + +static int rk_udphy_reset_assert_all(struct rk_udphy *udphy) +{ + return reset_control_bulk_assert(udphy->num_rsts, udphy->rsts); +} + +static int rk_udphy_reset_deassert_all(struct rk_udphy *udphy) +{ + return reset_control_bulk_deassert(udphy->num_rsts, udphy->rsts); +} + +static int rk_udphy_reset_deassert(struct rk_udphy *udphy, char *name) +{ + struct reset_control_bulk_data *list = udphy->rsts; + int idx; + + for (idx = 0; idx < udphy->num_rsts; idx++) { + if (!strcmp(list[idx].id, name)) + return reset_control_deassert(list[idx].rstc); + } + + return -EINVAL; +} + +static int rk_udphy_reset_init(struct rk_udphy *udphy, struct device *dev) +{ + const struct rk_udphy_cfg *cfg = udphy->cfgs; + int idx; + + udphy->num_rsts = cfg->num_rsts; + udphy->rsts = devm_kcalloc(dev, udphy->num_rsts, + sizeof(*udphy->rsts), GFP_KERNEL); + if (!udphy->rsts) + return -ENOMEM; + + for (idx = 0; idx < cfg->num_rsts; idx++) + udphy->rsts[idx].id = cfg->rst_list[idx]; + + return devm_reset_control_bulk_get_exclusive(dev, cfg->num_rsts, + udphy->rsts); +} + +static void rk_udphy_u3_port_disable(struct rk_udphy *udphy, u8 disable) +{ + const struct rk_udphy_cfg *cfg = udphy->cfgs; + const struct rk_udphy_grf_reg *preg; + + preg = udphy->id ? &cfg->grfcfg.usb3otg1_cfg : &cfg->grfcfg.usb3otg0_cfg; + rk_udphy_grfreg_write(udphy->usbgrf, preg, disable); +} + +static void rk_udphy_usb_bvalid_enable(struct rk_udphy *udphy, u8 enable) +{ + const struct rk_udphy_cfg *cfg = udphy->cfgs; + + rk_udphy_grfreg_write(udphy->u2phygrf, &cfg->grfcfg.bvalid_phy_con, enable); + rk_udphy_grfreg_write(udphy->u2phygrf, &cfg->grfcfg.bvalid_grf_con, enable); +} + +/* + * In usb/dp combo phy driver, here are 2 ways to mapping lanes. + * + * 1 Type-C Mapping table (DP_Alt_Mode V1.0b remove ABF pin mapping) + * --------------------------------------------------------------------------- + * Type-C Pin B11-B10 A2-A3 A11-A10 B2-B3 + * PHY Pad ln0(tx/rx) ln1(tx) ln2(tx/rx) ln3(tx) + * C/E(Normal) dpln3 dpln2 dpln0 dpln1 + * C/E(Flip ) dpln0 dpln1 dpln3 dpln2 + * D/F(Normal) usbrx usbtx dpln0 dpln1 + * D/F(Flip ) dpln0 dpln1 usbrx usbtx + * A(Normal ) dpln3 dpln1 dpln2 dpln0 + * A(Flip ) dpln2 dpln0 dpln3 dpln1 + * B(Normal ) usbrx usbtx dpln1 dpln0 + * B(Flip ) dpln1 dpln0 usbrx usbtx + * --------------------------------------------------------------------------- + * + * 2 Mapping the lanes in dtsi + * if all 4 lane assignment for dp function, define rockchip,dp-lane-mux = <x x x x>; + * sample as follow: + * --------------------------------------------------------------------------- + * B11-B10 A2-A3 A11-A10 B2-B3 + * rockchip,dp-lane-mux ln0(tx/rx) ln1(tx) ln2(tx/rx) ln3(tx) + * <0 1 2 3> dpln0 dpln1 dpln2 dpln3 + * <2 3 0 1> dpln2 dpln3 dpln0 dpln1 + * --------------------------------------------------------------------------- + * if 2 lane for dp function, 2 lane for usb function, define rockchip,dp-lane-mux = <x x>; + * sample as follow: + * --------------------------------------------------------------------------- + * B11-B10 A2-A3 A11-A10 B2-B3 + * rockchip,dp-lane-mux ln0(tx/rx) ln1(tx) ln2(tx/rx) ln3(tx) + * <0 1> dpln0 dpln1 usbrx usbtx + * <2 3> usbrx usbtx dpln0 dpln1 + * --------------------------------------------------------------------------- + */ + +static void rk_udphy_dplane_select(struct rk_udphy *udphy) +{ + const struct rk_udphy_cfg *cfg = udphy->cfgs; + u32 value = 0; + + switch (udphy->mode) { + case UDPHY_MODE_DP: + value |= 2 << udphy->dp_lane_sel[2] * 2; + value |= 3 << udphy->dp_lane_sel[3] * 2; + fallthrough; + + case UDPHY_MODE_DP_USB: + value |= 0 << udphy->dp_lane_sel[0] * 2; + value |= 1 << udphy->dp_lane_sel[1] * 2; + break; + + case UDPHY_MODE_USB: + break; + + default: + break; + } + + regmap_write(udphy->vogrf, cfg->vogrfcfg[udphy->id].dp_lane_reg, + ((DP_AUX_DIN_SEL | DP_AUX_DOUT_SEL | DP_LANE_SEL_ALL) << 16) | + FIELD_PREP(DP_AUX_DIN_SEL, udphy->dp_aux_din_sel) | + FIELD_PREP(DP_AUX_DOUT_SEL, udphy->dp_aux_dout_sel) | value); +} + +static int rk_udphy_dplane_get(struct rk_udphy *udphy) +{ + int dp_lanes; + + switch (udphy->mode) { + case UDPHY_MODE_DP: + dp_lanes = 4; + break; + + case UDPHY_MODE_DP_USB: + dp_lanes = 2; + break; + + case UDPHY_MODE_USB: + default: + dp_lanes = 0; + break; + } + + return dp_lanes; +} + +static void rk_udphy_dplane_enable(struct rk_udphy *udphy, int dp_lanes) +{ + u32 val = 0; + int i; + + for (i = 0; i < dp_lanes; i++) + val |= BIT(udphy->dp_lane_sel[i]); + + regmap_update_bits(udphy->pma_regmap, CMN_LANE_MUX_AND_EN_OFFSET, CMN_DP_LANE_EN_ALL, + FIELD_PREP(CMN_DP_LANE_EN_ALL, val)); + + if (!dp_lanes) + regmap_update_bits(udphy->pma_regmap, CMN_DP_RSTN_OFFSET, + CMN_DP_CMN_RSTN, FIELD_PREP(CMN_DP_CMN_RSTN, 0x0)); +} + +static void rk_udphy_dp_hpd_event_trigger(struct rk_udphy *udphy, bool hpd) +{ + const struct rk_udphy_cfg *cfg = udphy->cfgs; + + udphy->dp_sink_hpd_sel = true; + udphy->dp_sink_hpd_cfg = hpd; + + if (!udphy->dp_in_use) + return; + + rk_udphy_grfreg_write(udphy->vogrf, &cfg->vogrfcfg[udphy->id].hpd_trigger, hpd); +} + +static void rk_udphy_set_typec_default_mapping(struct rk_udphy *udphy) +{ + if (udphy->flip) { + udphy->dp_lane_sel[0] = 0; + udphy->dp_lane_sel[1] = 1; + udphy->dp_lane_sel[2] = 3; + udphy->dp_lane_sel[3] = 2; + udphy->lane_mux_sel[0] = PHY_LANE_MUX_DP; + udphy->lane_mux_sel[1] = PHY_LANE_MUX_DP; + udphy->lane_mux_sel[2] = PHY_LANE_MUX_USB; + udphy->lane_mux_sel[3] = PHY_LANE_MUX_USB; + udphy->dp_aux_dout_sel = PHY_AUX_DP_DATA_POL_INVERT; + udphy->dp_aux_din_sel = PHY_AUX_DP_DATA_POL_INVERT; + gpiod_set_value_cansleep(udphy->sbu1_dc_gpio, 1); + gpiod_set_value_cansleep(udphy->sbu2_dc_gpio, 0); + } else { + udphy->dp_lane_sel[0] = 2; + udphy->dp_lane_sel[1] = 3; + udphy->dp_lane_sel[2] = 1; + udphy->dp_lane_sel[3] = 0; + udphy->lane_mux_sel[0] = PHY_LANE_MUX_USB; + udphy->lane_mux_sel[1] = PHY_LANE_MUX_USB; + udphy->lane_mux_sel[2] = PHY_LANE_MUX_DP; + udphy->lane_mux_sel[3] = PHY_LANE_MUX_DP; + udphy->dp_aux_dout_sel = PHY_AUX_DP_DATA_POL_NORMAL; + udphy->dp_aux_din_sel = PHY_AUX_DP_DATA_POL_NORMAL; + gpiod_set_value_cansleep(udphy->sbu1_dc_gpio, 0); + gpiod_set_value_cansleep(udphy->sbu2_dc_gpio, 1); + } + + udphy->mode = UDPHY_MODE_DP_USB; +} + +static int rk_udphy_orien_sw_set(struct typec_switch_dev *sw, + enum typec_orientation orien) +{ + struct rk_udphy *udphy = typec_switch_get_drvdata(sw); + + mutex_lock(&udphy->mutex); + + if (orien == TYPEC_ORIENTATION_NONE) { + gpiod_set_value_cansleep(udphy->sbu1_dc_gpio, 0); + gpiod_set_value_cansleep(udphy->sbu2_dc_gpio, 0); + /* unattached */ + rk_udphy_usb_bvalid_enable(udphy, false); + goto unlock_ret; + } + + udphy->flip = (orien == TYPEC_ORIENTATION_REVERSE) ? true : false; + rk_udphy_set_typec_default_mapping(udphy); + rk_udphy_usb_bvalid_enable(udphy, true); + +unlock_ret: + mutex_unlock(&udphy->mutex); + return 0; +} + +static void rk_udphy_orien_switch_unregister(void *data) +{ + struct rk_udphy *udphy = data; + + typec_switch_unregister(udphy->sw); +} + +static int rk_udphy_setup_orien_switch(struct rk_udphy *udphy) +{ + struct typec_switch_desc sw_desc = { }; + + sw_desc.drvdata = udphy; + sw_desc.fwnode = dev_fwnode(udphy->dev); + sw_desc.set = rk_udphy_orien_sw_set; + + udphy->sw = typec_switch_register(udphy->dev, &sw_desc); + if (IS_ERR(udphy->sw)) { + dev_err(udphy->dev, "Error register typec orientation switch: %ld\n", + PTR_ERR(udphy->sw)); + return PTR_ERR(udphy->sw); + } + + return devm_add_action_or_reset(udphy->dev, + rk_udphy_orien_switch_unregister, udphy); +} + +static int rk_udphy_refclk_set(struct rk_udphy *udphy) +{ + unsigned long rate; + int ret; + + /* configure phy reference clock */ + rate = clk_get_rate(udphy->refclk); + dev_dbg(udphy->dev, "refclk freq %ld\n", rate); + + switch (rate) { + case 24000000: + ret = regmap_multi_reg_write(udphy->pma_regmap, rk_udphy_24m_refclk_cfg, + ARRAY_SIZE(rk_udphy_24m_refclk_cfg)); + if (ret) + return ret; + break; + + case 26000000: + /* register default is 26MHz */ + ret = regmap_multi_reg_write(udphy->pma_regmap, rk_udphy_26m_refclk_cfg, + ARRAY_SIZE(rk_udphy_26m_refclk_cfg)); + if (ret) + return ret; + break; + + default: + dev_err(udphy->dev, "unsupported refclk freq %ld\n", rate); + return -EINVAL; + } + + return 0; +} + +static int rk_udphy_status_check(struct rk_udphy *udphy) +{ + unsigned int val; + int ret; + + /* LCPLL check */ + if (udphy->mode & UDPHY_MODE_USB) { + ret = regmap_read_poll_timeout(udphy->pma_regmap, CMN_ANA_LCPLL_DONE_OFFSET, + val, (val & CMN_ANA_LCPLL_AFC_DONE) && + (val & CMN_ANA_LCPLL_LOCK_DONE), 200, 100000); + if (ret) { + dev_err(udphy->dev, "cmn ana lcpll lock timeout\n"); + /* + * If earlier software (U-Boot) enabled USB once already + * the PLL may have problems locking on the first try. + * It will be successful on the second try, so for the + * time being a -EPROBE_DEFER will solve the issue. + * + * This requires further investigation to understand the + * root cause, especially considering that the driver is + * asserting all reset lines at probe time. + */ + return -EPROBE_DEFER; + } + + if (!udphy->flip) { + ret = regmap_read_poll_timeout(udphy->pma_regmap, + TRSV_LN0_MON_RX_CDR_DONE_OFFSET, val, + val & TRSV_LN0_MON_RX_CDR_LOCK_DONE, + 200, 100000); + if (ret) + dev_err(udphy->dev, "trsv ln0 mon rx cdr lock timeout\n"); + } else { + ret = regmap_read_poll_timeout(udphy->pma_regmap, + TRSV_LN2_MON_RX_CDR_DONE_OFFSET, val, + val & TRSV_LN2_MON_RX_CDR_LOCK_DONE, + 200, 100000); + if (ret) + dev_err(udphy->dev, "trsv ln2 mon rx cdr lock timeout\n"); + } + } + + return 0; +} + +static int rk_udphy_init(struct rk_udphy *udphy) +{ + const struct rk_udphy_cfg *cfg = udphy->cfgs; + int ret; + + rk_udphy_reset_assert_all(udphy); + usleep_range(10000, 11000); + + /* enable rx lfps for usb */ + if (udphy->mode & UDPHY_MODE_USB) + rk_udphy_grfreg_write(udphy->udphygrf, &cfg->grfcfg.rx_lfps, true); + + /* Step 1: power on pma and deassert apb rstn */ + rk_udphy_grfreg_write(udphy->udphygrf, &cfg->grfcfg.low_pwrn, true); + + rk_udphy_reset_deassert(udphy, "pma_apb"); + rk_udphy_reset_deassert(udphy, "pcs_apb"); + + /* Step 2: set init sequence and phy refclk */ + ret = regmap_multi_reg_write(udphy->pma_regmap, rk_udphy_init_sequence, + ARRAY_SIZE(rk_udphy_init_sequence)); + if (ret) { + dev_err(udphy->dev, "init sequence set error %d\n", ret); + goto assert_resets; + } + + ret = rk_udphy_refclk_set(udphy); + if (ret) { + dev_err(udphy->dev, "refclk set error %d\n", ret); + goto assert_resets; + } + + /* Step 3: configure lane mux */ + regmap_update_bits(udphy->pma_regmap, CMN_LANE_MUX_AND_EN_OFFSET, + CMN_DP_LANE_MUX_ALL | CMN_DP_LANE_EN_ALL, + FIELD_PREP(CMN_DP_LANE_MUX_N(3), udphy->lane_mux_sel[3]) | + FIELD_PREP(CMN_DP_LANE_MUX_N(2), udphy->lane_mux_sel[2]) | + FIELD_PREP(CMN_DP_LANE_MUX_N(1), udphy->lane_mux_sel[1]) | + FIELD_PREP(CMN_DP_LANE_MUX_N(0), udphy->lane_mux_sel[0]) | + FIELD_PREP(CMN_DP_LANE_EN_ALL, 0)); + + /* Step 4: deassert init rstn and wait for 200ns from datasheet */ + if (udphy->mode & UDPHY_MODE_USB) + rk_udphy_reset_deassert(udphy, "init"); + + if (udphy->mode & UDPHY_MODE_DP) { + regmap_update_bits(udphy->pma_regmap, CMN_DP_RSTN_OFFSET, + CMN_DP_INIT_RSTN, + FIELD_PREP(CMN_DP_INIT_RSTN, 0x1)); + } + + udelay(1); + + /* Step 5: deassert cmn/lane rstn */ + if (udphy->mode & UDPHY_MODE_USB) { + rk_udphy_reset_deassert(udphy, "cmn"); + rk_udphy_reset_deassert(udphy, "lane"); + } + + /* Step 6: wait for lock done of pll */ + ret = rk_udphy_status_check(udphy); + if (ret) + goto assert_resets; + + return 0; + +assert_resets: + rk_udphy_reset_assert_all(udphy); + return ret; +} + +static int rk_udphy_setup(struct rk_udphy *udphy) +{ + int ret; + + ret = clk_bulk_prepare_enable(udphy->num_clks, udphy->clks); + if (ret) { + dev_err(udphy->dev, "failed to enable clk\n"); + return ret; + } + + ret = rk_udphy_init(udphy); + if (ret) { + dev_err(udphy->dev, "failed to init combophy\n"); + clk_bulk_disable_unprepare(udphy->num_clks, udphy->clks); + return ret; + } + + return 0; +} + +static void rk_udphy_disable(struct rk_udphy *udphy) +{ + clk_bulk_disable_unprepare(udphy->num_clks, udphy->clks); + rk_udphy_reset_assert_all(udphy); +} + +static int rk_udphy_parse_lane_mux_data(struct rk_udphy *udphy) +{ + int ret, i, num_lanes; + + num_lanes = device_property_count_u32(udphy->dev, "rockchip,dp-lane-mux"); + if (num_lanes < 0) { + dev_dbg(udphy->dev, "no dp-lane-mux, following dp alt mode\n"); + udphy->mode = UDPHY_MODE_USB; + return 0; + } + + if (num_lanes != 2 && num_lanes != 4) + return dev_err_probe(udphy->dev, -EINVAL, + "invalid number of lane mux\n"); + + ret = device_property_read_u32_array(udphy->dev, "rockchip,dp-lane-mux", + udphy->dp_lane_sel, num_lanes); + if (ret) + return dev_err_probe(udphy->dev, ret, "get dp lane mux failed\n"); + + for (i = 0; i < num_lanes; i++) { + int j; + + if (udphy->dp_lane_sel[i] > 3) + return dev_err_probe(udphy->dev, -EINVAL, + "lane mux between 0 and 3, exceeding the range\n"); + + udphy->lane_mux_sel[udphy->dp_lane_sel[i]] = PHY_LANE_MUX_DP; + + for (j = i + 1; j < num_lanes; j++) { + if (udphy->dp_lane_sel[i] == udphy->dp_lane_sel[j]) + return dev_err_probe(udphy->dev, -EINVAL, + "set repeat lane mux value\n"); + } + } + + udphy->mode = UDPHY_MODE_DP; + if (num_lanes == 2) { + udphy->mode |= UDPHY_MODE_USB; + udphy->flip = (udphy->lane_mux_sel[0] == PHY_LANE_MUX_DP); + } + + return 0; +} + +static int rk_udphy_get_initial_status(struct rk_udphy *udphy) +{ + int ret; + u32 value; + + ret = clk_bulk_prepare_enable(udphy->num_clks, udphy->clks); + if (ret) { + dev_err(udphy->dev, "failed to enable clk\n"); + return ret; + } + + rk_udphy_reset_deassert_all(udphy); + + regmap_read(udphy->pma_regmap, CMN_LANE_MUX_AND_EN_OFFSET, &value); + if (FIELD_GET(CMN_DP_LANE_MUX_ALL, value) && FIELD_GET(CMN_DP_LANE_EN_ALL, value)) + udphy->status = UDPHY_MODE_DP; + else + rk_udphy_disable(udphy); + + return 0; +} + +static int rk_udphy_parse_dt(struct rk_udphy *udphy) +{ + struct device *dev = udphy->dev; + struct device_node *np = dev_of_node(dev); + enum usb_device_speed maximum_speed; + int ret; + + udphy->u2phygrf = syscon_regmap_lookup_by_phandle(np, "rockchip,u2phy-grf"); + if (IS_ERR(udphy->u2phygrf)) + return dev_err_probe(dev, PTR_ERR(udphy->u2phygrf), "failed to get u2phy-grf\n"); + + udphy->udphygrf = syscon_regmap_lookup_by_phandle(np, "rockchip,usbdpphy-grf"); + if (IS_ERR(udphy->udphygrf)) + return dev_err_probe(dev, PTR_ERR(udphy->udphygrf), "failed to get usbdpphy-grf\n"); + + udphy->usbgrf = syscon_regmap_lookup_by_phandle(np, "rockchip,usb-grf"); + if (IS_ERR(udphy->usbgrf)) + return dev_err_probe(dev, PTR_ERR(udphy->usbgrf), "failed to get usb-grf\n"); + + udphy->vogrf = syscon_regmap_lookup_by_phandle(np, "rockchip,vo-grf"); + if (IS_ERR(udphy->vogrf)) + return dev_err_probe(dev, PTR_ERR(udphy->vogrf), "failed to get vo-grf\n"); + + ret = rk_udphy_parse_lane_mux_data(udphy); + if (ret) + return ret; + + udphy->sbu1_dc_gpio = devm_gpiod_get_optional(dev, "sbu1-dc", GPIOD_OUT_LOW); + if (IS_ERR(udphy->sbu1_dc_gpio)) + return PTR_ERR(udphy->sbu1_dc_gpio); + + udphy->sbu2_dc_gpio = devm_gpiod_get_optional(dev, "sbu2-dc", GPIOD_OUT_LOW); + if (IS_ERR(udphy->sbu2_dc_gpio)) + return PTR_ERR(udphy->sbu2_dc_gpio); + + if (device_property_present(dev, "maximum-speed")) { + maximum_speed = usb_get_maximum_speed(dev); + udphy->hs = maximum_speed <= USB_SPEED_HIGH ? true : false; + } + + ret = rk_udphy_clk_init(udphy, dev); + if (ret) + return ret; + + return rk_udphy_reset_init(udphy, dev); +} + +static int rk_udphy_power_on(struct rk_udphy *udphy, u8 mode) +{ + int ret; + + if (!(udphy->mode & mode)) { + dev_info(udphy->dev, "mode 0x%02x is not support\n", mode); + return 0; + } + + if (udphy->status == UDPHY_MODE_NONE) { + udphy->mode_change = false; + ret = rk_udphy_setup(udphy); + if (ret) + return ret; + + if (udphy->mode & UDPHY_MODE_USB) + rk_udphy_u3_port_disable(udphy, false); + } else if (udphy->mode_change) { + udphy->mode_change = false; + udphy->status = UDPHY_MODE_NONE; + if (udphy->mode == UDPHY_MODE_DP) + rk_udphy_u3_port_disable(udphy, true); + + rk_udphy_disable(udphy); + ret = rk_udphy_setup(udphy); + if (ret) + return ret; + } + + udphy->status |= mode; + + return 0; +} + +static void rk_udphy_power_off(struct rk_udphy *udphy, u8 mode) +{ + if (!(udphy->mode & mode)) { + dev_info(udphy->dev, "mode 0x%02x is not support\n", mode); + return; + } + + if (!udphy->status) + return; + + udphy->status &= ~mode; + + if (udphy->status == UDPHY_MODE_NONE) + rk_udphy_disable(udphy); +} + +static int rk_udphy_dp_phy_init(struct phy *phy) +{ + struct rk_udphy *udphy = phy_get_drvdata(phy); + + mutex_lock(&udphy->mutex); + + udphy->dp_in_use = true; + rk_udphy_dp_hpd_event_trigger(udphy, udphy->dp_sink_hpd_cfg); + + mutex_unlock(&udphy->mutex); + + return 0; +} + +static int rk_udphy_dp_phy_exit(struct phy *phy) +{ + struct rk_udphy *udphy = phy_get_drvdata(phy); + + mutex_lock(&udphy->mutex); + udphy->dp_in_use = false; + mutex_unlock(&udphy->mutex); + return 0; +} + +static int rk_udphy_dp_phy_power_on(struct phy *phy) +{ + struct rk_udphy *udphy = phy_get_drvdata(phy); + int ret, dp_lanes; + + mutex_lock(&udphy->mutex); + + dp_lanes = rk_udphy_dplane_get(udphy); + phy_set_bus_width(phy, dp_lanes); + + ret = rk_udphy_power_on(udphy, UDPHY_MODE_DP); + if (ret) + goto unlock; + + rk_udphy_dplane_enable(udphy, dp_lanes); + + rk_udphy_dplane_select(udphy); + +unlock: + mutex_unlock(&udphy->mutex); + /* + * If data send by aux channel too fast after phy power on, + * the aux may be not ready which will cause aux error. Adding + * delay to avoid this issue. + */ + usleep_range(10000, 11000); + return ret; +} + +static int rk_udphy_dp_phy_power_off(struct phy *phy) +{ + struct rk_udphy *udphy = phy_get_drvdata(phy); + + mutex_lock(&udphy->mutex); + rk_udphy_dplane_enable(udphy, 0); + rk_udphy_power_off(udphy, UDPHY_MODE_DP); + mutex_unlock(&udphy->mutex); + + return 0; +} + +static int rk_udphy_dp_phy_verify_link_rate(unsigned int link_rate) +{ + switch (link_rate) { + case 1620: + case 2700: + case 5400: + case 8100: + break; + + default: + return -EINVAL; + } + + return 0; +} + +static int rk_udphy_dp_phy_verify_config(struct rk_udphy *udphy, + struct phy_configure_opts_dp *dp) +{ + int i, ret; + + /* If changing link rate was required, verify it's supported. */ + ret = rk_udphy_dp_phy_verify_link_rate(dp->link_rate); + if (ret) + return ret; + + /* Verify lane count. */ + switch (dp->lanes) { + case 1: + case 2: + case 4: + /* valid lane count. */ + break; + + default: + return -EINVAL; + } + + /* + * If changing voltages is required, check swing and pre-emphasis + * levels, per-lane. + */ + if (dp->set_voltages) { + /* Lane count verified previously. */ + for (i = 0; i < dp->lanes; i++) { + if (dp->voltage[i] > 3 || dp->pre[i] > 3) + return -EINVAL; + + /* + * Sum of voltage swing and pre-emphasis levels cannot + * exceed 3. + */ + if (dp->voltage[i] + dp->pre[i] > 3) + return -EINVAL; + } + } + + return 0; +} + +static void rk_udphy_dp_set_voltage(struct rk_udphy *udphy, u8 bw, + u32 voltage, u32 pre, u32 lane) +{ + const struct rk_udphy_cfg *cfg = udphy->cfgs; + const struct rk_udphy_dp_tx_drv_ctrl (*dp_ctrl)[4]; + u32 offset = 0x800 * lane; + u32 val; + + if (udphy->mux) + dp_ctrl = cfg->dp_tx_ctrl_cfg_typec[bw]; + else + dp_ctrl = cfg->dp_tx_ctrl_cfg[bw]; + + val = dp_ctrl[voltage][pre].trsv_reg0204; + regmap_write(udphy->pma_regmap, 0x0810 + offset, val); + + val = dp_ctrl[voltage][pre].trsv_reg0205; + regmap_write(udphy->pma_regmap, 0x0814 + offset, val); + + val = dp_ctrl[voltage][pre].trsv_reg0206; + regmap_write(udphy->pma_regmap, 0x0818 + offset, val); + + val = dp_ctrl[voltage][pre].trsv_reg0207; + regmap_write(udphy->pma_regmap, 0x081c + offset, val); +} + +static int rk_udphy_dp_phy_configure(struct phy *phy, + union phy_configure_opts *opts) +{ + struct rk_udphy *udphy = phy_get_drvdata(phy); + struct phy_configure_opts_dp *dp = &opts->dp; + u32 i, val, lane; + int ret; + + ret = rk_udphy_dp_phy_verify_config(udphy, dp); + if (ret) + return ret; + + if (dp->set_rate) { + regmap_update_bits(udphy->pma_regmap, CMN_DP_RSTN_OFFSET, + CMN_DP_CMN_RSTN, FIELD_PREP(CMN_DP_CMN_RSTN, 0x0)); + + switch (dp->link_rate) { + case 1620: + udphy->bw = DP_BW_RBR; + break; + + case 2700: + udphy->bw = DP_BW_HBR; + break; + + case 5400: + udphy->bw = DP_BW_HBR2; + break; + + case 8100: + udphy->bw = DP_BW_HBR3; + break; + + default: + return -EINVAL; + } + + regmap_update_bits(udphy->pma_regmap, CMN_DP_LINK_OFFSET, CMN_DP_TX_LINK_BW, + FIELD_PREP(CMN_DP_TX_LINK_BW, udphy->bw)); + regmap_update_bits(udphy->pma_regmap, CMN_SSC_EN_OFFSET, CMN_ROPLL_SSC_EN, + FIELD_PREP(CMN_ROPLL_SSC_EN, dp->ssc)); + regmap_update_bits(udphy->pma_regmap, CMN_DP_RSTN_OFFSET, CMN_DP_CMN_RSTN, + FIELD_PREP(CMN_DP_CMN_RSTN, 0x1)); + + ret = regmap_read_poll_timeout(udphy->pma_regmap, CMN_ANA_ROPLL_DONE_OFFSET, val, + FIELD_GET(CMN_ANA_ROPLL_LOCK_DONE, val) && + FIELD_GET(CMN_ANA_ROPLL_AFC_DONE, val), + 0, 1000); + if (ret) { + dev_err(udphy->dev, "ROPLL is not lock, set_rate failed\n"); + return ret; + } + } + + if (dp->set_voltages) { + for (i = 0; i < dp->lanes; i++) { + lane = udphy->dp_lane_sel[i]; + switch (dp->link_rate) { + case 1620: + case 2700: + regmap_update_bits(udphy->pma_regmap, + TRSV_ANA_TX_CLK_OFFSET_N(lane), + LN_ANA_TX_SER_TXCLK_INV, + FIELD_PREP(LN_ANA_TX_SER_TXCLK_INV, + udphy->lane_mux_sel[lane])); + break; + + case 5400: + case 8100: + regmap_update_bits(udphy->pma_regmap, + TRSV_ANA_TX_CLK_OFFSET_N(lane), + LN_ANA_TX_SER_TXCLK_INV, + FIELD_PREP(LN_ANA_TX_SER_TXCLK_INV, 0x0)); + break; + } + + rk_udphy_dp_set_voltage(udphy, udphy->bw, dp->voltage[i], + dp->pre[i], lane); + } + } + + return 0; +} + +static const struct phy_ops rk_udphy_dp_phy_ops = { + .init = rk_udphy_dp_phy_init, + .exit = rk_udphy_dp_phy_exit, + .power_on = rk_udphy_dp_phy_power_on, + .power_off = rk_udphy_dp_phy_power_off, + .configure = rk_udphy_dp_phy_configure, + .owner = THIS_MODULE, +}; + +static int rk_udphy_usb3_phy_init(struct phy *phy) +{ + struct rk_udphy *udphy = phy_get_drvdata(phy); + int ret = 0; + + mutex_lock(&udphy->mutex); + /* DP only or high-speed, disable U3 port */ + if (!(udphy->mode & UDPHY_MODE_USB) || udphy->hs) { + rk_udphy_u3_port_disable(udphy, true); + goto unlock; + } + + ret = rk_udphy_power_on(udphy, UDPHY_MODE_USB); + +unlock: + mutex_unlock(&udphy->mutex); + return ret; +} + +static int rk_udphy_usb3_phy_exit(struct phy *phy) +{ + struct rk_udphy *udphy = phy_get_drvdata(phy); + + mutex_lock(&udphy->mutex); + /* DP only or high-speed */ + if (!(udphy->mode & UDPHY_MODE_USB) || udphy->hs) + goto unlock; + + rk_udphy_power_off(udphy, UDPHY_MODE_USB); + +unlock: + mutex_unlock(&udphy->mutex); + return 0; +} + +static const struct phy_ops rk_udphy_usb3_phy_ops = { + .init = rk_udphy_usb3_phy_init, + .exit = rk_udphy_usb3_phy_exit, + .owner = THIS_MODULE, +}; + +static int rk_udphy_typec_mux_set(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct rk_udphy *udphy = typec_mux_get_drvdata(mux); + u8 mode; + + mutex_lock(&udphy->mutex); + + switch (state->mode) { + case TYPEC_DP_STATE_C: + case TYPEC_DP_STATE_E: + udphy->lane_mux_sel[0] = PHY_LANE_MUX_DP; + udphy->lane_mux_sel[1] = PHY_LANE_MUX_DP; + udphy->lane_mux_sel[2] = PHY_LANE_MUX_DP; + udphy->lane_mux_sel[3] = PHY_LANE_MUX_DP; + mode = UDPHY_MODE_DP; + break; + + case TYPEC_DP_STATE_D: + default: + if (udphy->flip) { + udphy->lane_mux_sel[0] = PHY_LANE_MUX_DP; + udphy->lane_mux_sel[1] = PHY_LANE_MUX_DP; + udphy->lane_mux_sel[2] = PHY_LANE_MUX_USB; + udphy->lane_mux_sel[3] = PHY_LANE_MUX_USB; + } else { + udphy->lane_mux_sel[0] = PHY_LANE_MUX_USB; + udphy->lane_mux_sel[1] = PHY_LANE_MUX_USB; + udphy->lane_mux_sel[2] = PHY_LANE_MUX_DP; + udphy->lane_mux_sel[3] = PHY_LANE_MUX_DP; + } + mode = UDPHY_MODE_DP_USB; + break; + } + + if (state->alt && state->alt->svid == USB_TYPEC_DP_SID) { + struct typec_displayport_data *data = state->data; + + if (!data) { + rk_udphy_dp_hpd_event_trigger(udphy, false); + } else if (data->status & DP_STATUS_IRQ_HPD) { + rk_udphy_dp_hpd_event_trigger(udphy, false); + usleep_range(750, 800); + rk_udphy_dp_hpd_event_trigger(udphy, true); + } else if (data->status & DP_STATUS_HPD_STATE) { + if (udphy->mode != mode) { + udphy->mode = mode; + udphy->mode_change = true; + } + rk_udphy_dp_hpd_event_trigger(udphy, true); + } else { + rk_udphy_dp_hpd_event_trigger(udphy, false); + } + } + + mutex_unlock(&udphy->mutex); + return 0; +} + +static void rk_udphy_typec_mux_unregister(void *data) +{ + struct rk_udphy *udphy = data; + + typec_mux_unregister(udphy->mux); +} + +static int rk_udphy_setup_typec_mux(struct rk_udphy *udphy) +{ + struct typec_mux_desc mux_desc = {}; + + mux_desc.drvdata = udphy; + mux_desc.fwnode = dev_fwnode(udphy->dev); + mux_desc.set = rk_udphy_typec_mux_set; + + udphy->mux = typec_mux_register(udphy->dev, &mux_desc); + if (IS_ERR(udphy->mux)) { + dev_err(udphy->dev, "Error register typec mux: %ld\n", + PTR_ERR(udphy->mux)); + return PTR_ERR(udphy->mux); + } + + return devm_add_action_or_reset(udphy->dev, rk_udphy_typec_mux_unregister, + udphy); +} + +static const struct regmap_config rk_udphy_pma_regmap_cfg = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .fast_io = true, + .max_register = 0x20dc, +}; + +static struct phy *rk_udphy_phy_xlate(struct device *dev, const struct of_phandle_args *args) +{ + struct rk_udphy *udphy = dev_get_drvdata(dev); + + if (args->args_count == 0) + return ERR_PTR(-EINVAL); + + switch (args->args[0]) { + case PHY_TYPE_USB3: + return udphy->phy_u3; + case PHY_TYPE_DP: + return udphy->phy_dp; + } + + return ERR_PTR(-EINVAL); +} + +static int rk_udphy_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct phy_provider *phy_provider; + struct resource *res; + struct rk_udphy *udphy; + void __iomem *base; + int id, ret; + + udphy = devm_kzalloc(dev, sizeof(*udphy), GFP_KERNEL); + if (!udphy) + return -ENOMEM; + + udphy->cfgs = device_get_match_data(dev); + if (!udphy->cfgs) + return dev_err_probe(dev, -EINVAL, "missing match data\n"); + + base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(base)) + return PTR_ERR(base); + + /* find the phy-id from the io address */ + udphy->id = -ENODEV; + for (id = 0; id < udphy->cfgs->num_phys; id++) { + if (res->start == udphy->cfgs->phy_ids[id]) { + udphy->id = id; + break; + } + } + + if (udphy->id < 0) + return dev_err_probe(dev, -ENODEV, "no matching device found\n"); + + udphy->pma_regmap = devm_regmap_init_mmio(dev, base + UDPHY_PMA, + &rk_udphy_pma_regmap_cfg); + if (IS_ERR(udphy->pma_regmap)) + return PTR_ERR(udphy->pma_regmap); + + udphy->dev = dev; + ret = rk_udphy_parse_dt(udphy); + if (ret) + return ret; + + ret = rk_udphy_get_initial_status(udphy); + if (ret) + return ret; + + mutex_init(&udphy->mutex); + platform_set_drvdata(pdev, udphy); + + if (device_property_present(dev, "orientation-switch")) { + ret = rk_udphy_setup_orien_switch(udphy); + if (ret) + return ret; + } + + if (device_property_present(dev, "mode-switch")) { + ret = rk_udphy_setup_typec_mux(udphy); + if (ret) + return ret; + } + + udphy->phy_u3 = devm_phy_create(dev, dev->of_node, &rk_udphy_usb3_phy_ops); + if (IS_ERR(udphy->phy_u3)) { + ret = PTR_ERR(udphy->phy_u3); + return dev_err_probe(dev, ret, "failed to create USB3 phy\n"); + } + phy_set_drvdata(udphy->phy_u3, udphy); + + udphy->phy_dp = devm_phy_create(dev, dev->of_node, &rk_udphy_dp_phy_ops); + if (IS_ERR(udphy->phy_dp)) { + ret = PTR_ERR(udphy->phy_dp); + return dev_err_probe(dev, ret, "failed to create DP phy\n"); + } + phy_set_bus_width(udphy->phy_dp, rk_udphy_dplane_get(udphy)); + udphy->phy_dp->attrs.max_link_rate = 8100; + phy_set_drvdata(udphy->phy_dp, udphy); + + phy_provider = devm_of_phy_provider_register(dev, rk_udphy_phy_xlate); + if (IS_ERR(phy_provider)) { + ret = PTR_ERR(phy_provider); + return dev_err_probe(dev, ret, "failed to register phy provider\n"); + } + + return 0; +} + +static int __maybe_unused rk_udphy_resume(struct device *dev) +{ + struct rk_udphy *udphy = dev_get_drvdata(dev); + + if (udphy->dp_sink_hpd_sel) + rk_udphy_dp_hpd_event_trigger(udphy, udphy->dp_sink_hpd_cfg); + + return 0; +} + +static const struct dev_pm_ops rk_udphy_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(NULL, rk_udphy_resume) +}; + +static const char * const rk_udphy_rst_list[] = { + "init", "cmn", "lane", "pcs_apb", "pma_apb" +}; + +static const struct rk_udphy_cfg rk3588_udphy_cfgs = { + .num_phys = 2, + .phy_ids = { + 0xfed80000, + 0xfed90000, + }, + .num_rsts = ARRAY_SIZE(rk_udphy_rst_list), + .rst_list = rk_udphy_rst_list, + .grfcfg = { + /* u2phy-grf */ + .bvalid_phy_con = RK_UDPHY_GEN_GRF_REG(0x0008, 1, 0, 0x2, 0x3), + .bvalid_grf_con = RK_UDPHY_GEN_GRF_REG(0x0010, 3, 2, 0x2, 0x3), + + /* usb-grf */ + .usb3otg0_cfg = RK_UDPHY_GEN_GRF_REG(0x001c, 15, 0, 0x1100, 0x0188), + .usb3otg1_cfg = RK_UDPHY_GEN_GRF_REG(0x0034, 15, 0, 0x1100, 0x0188), + + /* usbdpphy-grf */ + .low_pwrn = RK_UDPHY_GEN_GRF_REG(0x0004, 13, 13, 0, 1), + .rx_lfps = RK_UDPHY_GEN_GRF_REG(0x0004, 14, 14, 0, 1), + }, + .vogrfcfg = { + { + .hpd_trigger = RK_UDPHY_GEN_GRF_REG(0x0000, 11, 10, 1, 3), + .dp_lane_reg = 0x0000, + }, + { + .hpd_trigger = RK_UDPHY_GEN_GRF_REG(0x0008, 11, 10, 1, 3), + .dp_lane_reg = 0x0008, + }, + }, + .dp_tx_ctrl_cfg = { + rk3588_dp_tx_drv_ctrl_rbr_hbr, + rk3588_dp_tx_drv_ctrl_rbr_hbr, + rk3588_dp_tx_drv_ctrl_hbr2, + rk3588_dp_tx_drv_ctrl_hbr3, + }, + .dp_tx_ctrl_cfg_typec = { + rk3588_dp_tx_drv_ctrl_rbr_hbr_typec, + rk3588_dp_tx_drv_ctrl_rbr_hbr_typec, + rk3588_dp_tx_drv_ctrl_hbr2, + rk3588_dp_tx_drv_ctrl_hbr3, + }, +}; + +static const struct of_device_id rk_udphy_dt_match[] = { + { + .compatible = "rockchip,rk3588-usbdp-phy", + .data = &rk3588_udphy_cfgs + }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, rk_udphy_dt_match); + +static struct platform_driver rk_udphy_driver = { + .probe = rk_udphy_probe, + .driver = { + .name = "rockchip-usbdp-phy", + .of_match_table = rk_udphy_dt_match, + .pm = &rk_udphy_pm_ops, + }, +}; +module_platform_driver(rk_udphy_driver); + +MODULE_AUTHOR("Frank Wang <frank.wang@rock-chips.com>"); +MODULE_AUTHOR("Zhang Yubing <yubing.zhang@rock-chips.com>"); +MODULE_DESCRIPTION("Rockchip USBDP Combo PHY driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/samsung/Makefile b/drivers/phy/samsung/Makefile index afb34a153e34..fea1f96d0e43 100644 --- a/drivers/phy/samsung/Makefile +++ b/drivers/phy/samsung/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_PHY_EXYNOS_DP_VIDEO) += phy-exynos-dp-video.o obj-$(CONFIG_PHY_EXYNOS_MIPI_VIDEO) += phy-exynos-mipi-video.o obj-$(CONFIG_PHY_EXYNOS_PCIE) += phy-exynos-pcie.o obj-$(CONFIG_PHY_SAMSUNG_UFS) += phy-exynos-ufs.o +phy-exynos-ufs-y += phy-gs101-ufs.o phy-exynos-ufs-y += phy-samsung-ufs.o phy-exynos-ufs-y += phy-exynos7-ufs.o phy-exynos-ufs-y += phy-exynosautov9-ufs.o diff --git a/drivers/phy/samsung/phy-exynos7-ufs.c b/drivers/phy/samsung/phy-exynos7-ufs.c index a982e7c128c5..15eec1d9e0e0 100644 --- a/drivers/phy/samsung/phy-exynos7-ufs.c +++ b/drivers/phy/samsung/phy-exynos7-ufs.c @@ -82,4 +82,5 @@ const struct samsung_ufs_phy_drvdata exynos7_ufs_phy = { .clk_list = exynos7_ufs_phy_clks, .num_clks = ARRAY_SIZE(exynos7_ufs_phy_clks), .cdr_lock_status_offset = EXYNOS7_EMBEDDED_COMBO_PHY_CDR_LOCK_STATUS, + .wait_for_cdr = samsung_ufs_phy_wait_for_lock_acq, }; diff --git a/drivers/phy/samsung/phy-exynosautov9-ufs.c b/drivers/phy/samsung/phy-exynosautov9-ufs.c index 49e2bcbef0b4..9c3e030f07ba 100644 --- a/drivers/phy/samsung/phy-exynosautov9-ufs.c +++ b/drivers/phy/samsung/phy-exynosautov9-ufs.c @@ -71,4 +71,5 @@ const struct samsung_ufs_phy_drvdata exynosautov9_ufs_phy = { .clk_list = exynosautov9_ufs_phy_clks, .num_clks = ARRAY_SIZE(exynosautov9_ufs_phy_clks), .cdr_lock_status_offset = EXYNOSAUTOV9_EMBEDDED_COMBO_PHY_CDR_LOCK_STATUS, + .wait_for_cdr = samsung_ufs_phy_wait_for_lock_acq, }; diff --git a/drivers/phy/samsung/phy-fsd-ufs.c b/drivers/phy/samsung/phy-fsd-ufs.c index d36cabd53434..f2361746db0e 100644 --- a/drivers/phy/samsung/phy-fsd-ufs.c +++ b/drivers/phy/samsung/phy-fsd-ufs.c @@ -60,4 +60,5 @@ const struct samsung_ufs_phy_drvdata fsd_ufs_phy = { .clk_list = fsd_ufs_phy_clks, .num_clks = ARRAY_SIZE(fsd_ufs_phy_clks), .cdr_lock_status_offset = FSD_EMBEDDED_COMBO_PHY_CDR_LOCK_STATUS, + .wait_for_cdr = samsung_ufs_phy_wait_for_lock_acq, }; diff --git a/drivers/phy/samsung/phy-gs101-ufs.c b/drivers/phy/samsung/phy-gs101-ufs.c new file mode 100644 index 000000000000..17b798da5b57 --- /dev/null +++ b/drivers/phy/samsung/phy-gs101-ufs.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * UFS PHY driver data for Google Tensor gs101 SoC + * + * Copyright (C) 2024 Linaro Ltd + * Author: Peter Griffin <peter.griffin@linaro.org> + */ + +#include "phy-samsung-ufs.h" + +#define TENSOR_GS101_PHY_CTRL 0x3ec8 +#define TENSOR_GS101_PHY_CTRL_MASK 0x1 +#define TENSOR_GS101_PHY_CTRL_EN BIT(0) +#define PHY_GS101_LANE_OFFSET 0x200 +#define TRSV_REG338 0x338 +#define LN0_MON_RX_CAL_DONE BIT(3) +#define TRSV_REG339 0x339 +#define LN0_MON_RX_CDR_FLD_CK_MODE_DONE BIT(3) +#define TRSV_REG222 0x222 +#define LN0_OVRD_RX_CDR_EN BIT(4) +#define LN0_RX_CDR_EN BIT(3) + +#define PHY_PMA_TRSV_ADDR(reg, lane) (PHY_APB_ADDR((reg) + \ + ((lane) * PHY_GS101_LANE_OFFSET))) + +#define PHY_TRSV_REG_CFG_GS101(o, v, d) \ + PHY_TRSV_REG_CFG_OFFSET(o, v, d, PHY_GS101_LANE_OFFSET) + +/* Calibration for phy initialization */ +static const struct samsung_ufs_phy_cfg tensor_gs101_pre_init_cfg[] = { + PHY_COMN_REG_CFG(0x43, 0x10, PWR_MODE_ANY), + PHY_COMN_REG_CFG(0x3C, 0x14, PWR_MODE_ANY), + PHY_COMN_REG_CFG(0x46, 0x48, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x200, 0x00, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x201, 0x06, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x202, 0x06, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x203, 0x0a, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x204, 0x00, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x205, 0x11, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x207, 0x0c, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2E1, 0xc0, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x22D, 0xb8, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x234, 0x60, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x238, 0x13, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x239, 0x48, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x23A, 0x01, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x23B, 0x25, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x23C, 0x2a, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x23D, 0x01, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x23E, 0x13, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x23F, 0x13, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x240, 0x4a, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x243, 0x40, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x244, 0x02, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x25D, 0x00, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x25E, 0x3f, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x25F, 0xff, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x273, 0x33, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x274, 0x50, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x284, 0x02, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x285, 0x02, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2A2, 0x04, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x25D, 0x01, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2FA, 0x01, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x286, 0x03, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x287, 0x03, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x288, 0x03, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x289, 0x03, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2B3, 0x04, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2B6, 0x0b, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2B7, 0x0b, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2B8, 0x0b, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2B9, 0x0b, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2BA, 0x0b, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2BB, 0x06, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2BC, 0x06, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2BD, 0x06, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x29E, 0x06, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2E4, 0x1a, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2ED, 0x25, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x269, 0x1a, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x2F4, 0x2f, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x34B, 0x01, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x34C, 0x23, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x34D, 0x23, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x34E, 0x45, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x34F, 0x00, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x350, 0x31, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x351, 0x00, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x352, 0x02, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x353, 0x00, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x354, 0x01, PWR_MODE_ANY), + PHY_COMN_REG_CFG(0x43, 0x18, PWR_MODE_ANY), + PHY_COMN_REG_CFG(0x43, 0x00, PWR_MODE_ANY), + END_UFS_PHY_CFG, +}; + +static const struct samsung_ufs_phy_cfg tensor_gs101_pre_pwr_hs_config[] = { + PHY_TRSV_REG_CFG_GS101(0x369, 0x11, PWR_MODE_ANY), + PHY_TRSV_REG_CFG_GS101(0x246, 0x03, PWR_MODE_ANY), +}; + +/* Calibration for HS mode series A/B */ +static const struct samsung_ufs_phy_cfg tensor_gs101_post_pwr_hs_config[] = { + PHY_COMN_REG_CFG(0x8, 0x60, PWR_MODE_PWM_ANY), + PHY_TRSV_REG_CFG_GS101(0x222, 0x08, PWR_MODE_PWM_ANY), + PHY_TRSV_REG_CFG_GS101(0x246, 0x01, PWR_MODE_ANY), + END_UFS_PHY_CFG, +}; + +static const struct samsung_ufs_phy_cfg *tensor_gs101_ufs_phy_cfgs[CFG_TAG_MAX] = { + [CFG_PRE_INIT] = tensor_gs101_pre_init_cfg, + [CFG_PRE_PWR_HS] = tensor_gs101_pre_pwr_hs_config, + [CFG_POST_PWR_HS] = tensor_gs101_post_pwr_hs_config, +}; + +static const char * const tensor_gs101_ufs_phy_clks[] = { + "ref_clk", +}; + +static int gs101_phy_wait_for_calibration(struct phy *phy, u8 lane) +{ + struct samsung_ufs_phy *ufs_phy = get_samsung_ufs_phy(phy); + const unsigned int timeout_us = 40000; + const unsigned int sleep_us = 40; + u32 val; + u32 off; + int err; + + off = PHY_PMA_TRSV_ADDR(TRSV_REG338, lane); + + err = readl_poll_timeout(ufs_phy->reg_pma + off, + val, (val & LN0_MON_RX_CAL_DONE), + sleep_us, timeout_us); + + if (err) { + dev_err(ufs_phy->dev, + "failed to get phy cal done %d\n", err); + } + + return err; +} + +#define DELAY_IN_US 40 +#define RETRY_CNT 100 +static int gs101_phy_wait_for_cdr_lock(struct phy *phy, u8 lane) +{ + struct samsung_ufs_phy *ufs_phy = get_samsung_ufs_phy(phy); + u32 val; + int i; + + for (i = 0; i < RETRY_CNT; i++) { + udelay(DELAY_IN_US); + val = readl(ufs_phy->reg_pma + + PHY_PMA_TRSV_ADDR(TRSV_REG339, lane)); + + if (val & LN0_MON_RX_CDR_FLD_CK_MODE_DONE) + return 0; + + udelay(DELAY_IN_US); + /* Override and enable clock data recovery */ + writel(LN0_OVRD_RX_CDR_EN, ufs_phy->reg_pma + + PHY_PMA_TRSV_ADDR(TRSV_REG222, lane)); + writel(LN0_OVRD_RX_CDR_EN | LN0_RX_CDR_EN, + ufs_phy->reg_pma + PHY_PMA_TRSV_ADDR(TRSV_REG222, lane)); + } + dev_err(ufs_phy->dev, "failed to get cdr lock\n"); + return -ETIMEDOUT; +} + +const struct samsung_ufs_phy_drvdata tensor_gs101_ufs_phy = { + .cfgs = tensor_gs101_ufs_phy_cfgs, + .isol = { + .offset = TENSOR_GS101_PHY_CTRL, + .mask = TENSOR_GS101_PHY_CTRL_MASK, + .en = TENSOR_GS101_PHY_CTRL_EN, + }, + .clk_list = tensor_gs101_ufs_phy_clks, + .num_clks = ARRAY_SIZE(tensor_gs101_ufs_phy_clks), + .wait_for_cal = gs101_phy_wait_for_calibration, + .wait_for_cdr = gs101_phy_wait_for_cdr_lock, +}; diff --git a/drivers/phy/samsung/phy-samsung-ufs.c b/drivers/phy/samsung/phy-samsung-ufs.c index 183c88e3d1ec..6c5d41552649 100644 --- a/drivers/phy/samsung/phy-samsung-ufs.c +++ b/drivers/phy/samsung/phy-samsung-ufs.c @@ -13,11 +13,11 @@ #include <linux/of.h> #include <linux/io.h> #include <linux/iopoll.h> -#include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/phy/phy.h> #include <linux/platform_device.h> #include <linux/regmap.h> +#include <linux/soc/samsung/exynos-pmu.h> #include "phy-samsung-ufs.h" @@ -45,7 +45,7 @@ static void samsung_ufs_phy_config(struct samsung_ufs_phy *phy, } } -static int samsung_ufs_phy_wait_for_lock_acq(struct phy *phy) +int samsung_ufs_phy_wait_for_lock_acq(struct phy *phy, u8 lane) { struct samsung_ufs_phy *ufs_phy = get_samsung_ufs_phy(phy); const unsigned int timeout_us = 100000; @@ -97,8 +97,21 @@ static int samsung_ufs_phy_calibrate(struct phy *phy) } } - if (ufs_phy->ufs_phy_state == CFG_POST_PWR_HS) - err = samsung_ufs_phy_wait_for_lock_acq(phy); + for_each_phy_lane(ufs_phy, i) { + if (ufs_phy->ufs_phy_state == CFG_PRE_INIT && + ufs_phy->drvdata->wait_for_cal) { + err = ufs_phy->drvdata->wait_for_cal(phy, i); + if (err) + goto out; + } + + if (ufs_phy->ufs_phy_state == CFG_POST_PWR_HS && + ufs_phy->drvdata->wait_for_cdr) { + err = ufs_phy->drvdata->wait_for_cdr(phy, i); + if (err) + goto out; + } + } /** * In Samsung ufshci, PHY need to be calibrated at different @@ -255,8 +268,8 @@ static int samsung_ufs_phy_probe(struct platform_device *pdev) goto out; } - phy->reg_pmu = syscon_regmap_lookup_by_phandle( - dev->of_node, "samsung,pmu-syscon"); + phy->reg_pmu = exynos_get_pmu_regmap_by_phandle(dev->of_node, + "samsung,pmu-syscon"); if (IS_ERR(phy->reg_pmu)) { err = PTR_ERR(phy->reg_pmu); dev_err(dev, "failed syscon remap for pmu\n"); @@ -302,6 +315,9 @@ out: static const struct of_device_id samsung_ufs_phy_match[] = { { + .compatible = "google,gs101-ufs-phy", + .data = &tensor_gs101_ufs_phy, + }, { .compatible = "samsung,exynos7-ufs-phy", .data = &exynos7_ufs_phy, }, { diff --git a/drivers/phy/samsung/phy-samsung-ufs.h b/drivers/phy/samsung/phy-samsung-ufs.h index e122960cfee8..9b7deef6e10f 100644 --- a/drivers/phy/samsung/phy-samsung-ufs.h +++ b/drivers/phy/samsung/phy-samsung-ufs.h @@ -112,6 +112,9 @@ struct samsung_ufs_phy_drvdata { const char * const *clk_list; int num_clks; u32 cdr_lock_status_offset; + /* SoC's specific operations */ + int (*wait_for_cal)(struct phy *phy, u8 lane); + int (*wait_for_cdr)(struct phy *phy, u8 lane); }; struct samsung_ufs_phy { @@ -139,8 +142,11 @@ static inline void samsung_ufs_phy_ctrl_isol( phy->isol.mask, isol ? 0 : phy->isol.en); } +int samsung_ufs_phy_wait_for_lock_acq(struct phy *phy, u8 lane); + extern const struct samsung_ufs_phy_drvdata exynos7_ufs_phy; extern const struct samsung_ufs_phy_drvdata exynosautov9_ufs_phy; extern const struct samsung_ufs_phy_drvdata fsd_ufs_phy; +extern const struct samsung_ufs_phy_drvdata tensor_gs101_ufs_phy; #endif /* _PHY_SAMSUNG_UFS_ */ diff --git a/drivers/phy/xilinx/phy-zynqmp.c b/drivers/phy/xilinx/phy-zynqmp.c index f72c5257d712..dc8319bda43d 100644 --- a/drivers/phy/xilinx/phy-zynqmp.c +++ b/drivers/phy/xilinx/phy-zynqmp.c @@ -995,15 +995,13 @@ static int xpsgtr_probe(struct platform_device *pdev) return 0; } -static int xpsgtr_remove(struct platform_device *pdev) +static void xpsgtr_remove(struct platform_device *pdev) { struct xpsgtr_dev *gtr_dev = platform_get_drvdata(pdev); pm_runtime_disable(gtr_dev->dev); pm_runtime_put_noidle(gtr_dev->dev); pm_runtime_set_suspended(gtr_dev->dev); - - return 0; } static const struct of_device_id xpsgtr_of_match[] = { @@ -1015,7 +1013,7 @@ MODULE_DEVICE_TABLE(of, xpsgtr_of_match); static struct platform_driver xpsgtr_driver = { .probe = xpsgtr_probe, - .remove = xpsgtr_remove, + .remove_new = xpsgtr_remove, .driver = { .name = "xilinx-psgtr", .of_match_table = xpsgtr_of_match, diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c index 1489191a213f..7178a38475cc 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c @@ -244,6 +244,10 @@ static const char * const irq_type_names[] = { [IRQ_TYPE_LEVEL_LOW] = "level-low", }; +static bool persist_gpio_outputs; +module_param(persist_gpio_outputs, bool, 0644); +MODULE_PARM_DESC(persist_gpio_outputs, "Enable GPIO_OUT persistence when pin is freed"); + static inline u32 bcm2835_gpio_rd(struct bcm2835_pinctrl *pc, unsigned reg) { return readl(pc->base + reg); @@ -926,6 +930,13 @@ static int bcm2835_pmx_free(struct pinctrl_dev *pctldev, unsigned offset) { struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); + enum bcm2835_fsel fsel = bcm2835_pinctrl_fsel_get(pc, offset); + + if (fsel == BCM2835_FSEL_GPIO_IN) + return 0; + + if (persist_gpio_outputs && fsel == BCM2835_FSEL_GPIO_OUT) + return 0; /* disable by setting to GPIO_IN */ bcm2835_pinctrl_fsel_set(pc, offset, BCM2835_FSEL_GPIO_IN); @@ -970,10 +981,7 @@ static void bcm2835_pmx_gpio_disable_free(struct pinctrl_dev *pctldev, struct pinctrl_gpio_range *range, unsigned offset) { - struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); - - /* disable by setting to GPIO_IN */ - bcm2835_pinctrl_fsel_set(pc, offset, BCM2835_FSEL_GPIO_IN); + bcm2835_pmx_free(pctldev, offset); } static int bcm2835_pmx_gpio_set_direction(struct pinctrl_dev *pctldev, @@ -1003,8 +1011,27 @@ static const struct pinmux_ops bcm2835_pmx_ops = { static int bcm2835_pinconf_get(struct pinctrl_dev *pctldev, unsigned pin, unsigned long *config) { - /* No way to read back config in HW */ - return -ENOTSUPP; + enum pin_config_param param = pinconf_to_config_param(*config); + struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); + enum bcm2835_fsel fsel = bcm2835_pinctrl_fsel_get(pc, pin); + u32 val; + + /* No way to read back bias config in HW */ + + switch (param) { + case PIN_CONFIG_OUTPUT: + if (fsel != BCM2835_FSEL_GPIO_OUT) + return -EINVAL; + + val = bcm2835_gpio_get_bit(pc, GPLEV0, pin); + *config = pinconf_to_config_packed(param, val); + break; + + default: + return -ENOTSUPP; + } + + return 0; } static void bcm2835_pull_config_set(struct bcm2835_pinctrl *pc, @@ -1079,6 +1106,45 @@ static const struct pinconf_ops bcm2835_pinconf_ops = { .pin_config_set = bcm2835_pinconf_set, }; +static int bcm2711_pinconf_get(struct pinctrl_dev *pctldev, unsigned pin, + unsigned long *config) +{ + struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); + enum pin_config_param param = pinconf_to_config_param(*config); + u32 offset, shift, val; + + offset = PUD_2711_REG_OFFSET(pin); + shift = PUD_2711_REG_SHIFT(pin); + val = bcm2835_gpio_rd(pc, GP_GPIO_PUP_PDN_CNTRL_REG0 + (offset * 4)); + + switch (param) { + case PIN_CONFIG_BIAS_DISABLE: + if (((val >> shift) & PUD_2711_MASK) != BCM2711_PULL_NONE) + return -EINVAL; + + break; + + case PIN_CONFIG_BIAS_PULL_UP: + if (((val >> shift) & PUD_2711_MASK) != BCM2711_PULL_UP) + return -EINVAL; + + *config = pinconf_to_config_packed(param, 50000); + break; + + case PIN_CONFIG_BIAS_PULL_DOWN: + if (((val >> shift) & PUD_2711_MASK) != BCM2711_PULL_DOWN) + return -EINVAL; + + *config = pinconf_to_config_packed(param, 50000); + break; + + default: + return bcm2835_pinconf_get(pctldev, pin, config); + } + + return 0; +} + static void bcm2711_pull_config_set(struct bcm2835_pinctrl *pc, unsigned int pin, unsigned int arg) { @@ -1146,7 +1212,7 @@ static int bcm2711_pinconf_set(struct pinctrl_dev *pctldev, static const struct pinconf_ops bcm2711_pinconf_ops = { .is_generic = true, - .pin_config_get = bcm2835_pinconf_get, + .pin_config_get = bcm2711_pinconf_get, .pin_config_set = bcm2711_pinconf_set, }; @@ -1361,6 +1427,9 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev) goto out_remove; } + dev_info(dev, "GPIO_OUT persistence: %s\n", + persist_gpio_outputs ? "yes" : "no"); + return 0; out_remove: diff --git a/drivers/pinctrl/freescale/pinctrl-imx8ulp.c b/drivers/pinctrl/freescale/pinctrl-imx8ulp.c index 2e86ca9fc7ac..5632c7285147 100644 --- a/drivers/pinctrl/freescale/pinctrl-imx8ulp.c +++ b/drivers/pinctrl/freescale/pinctrl-imx8ulp.c @@ -252,6 +252,7 @@ static const struct of_device_id imx8ulp_pinctrl_of_match[] = { { .compatible = "fsl,imx8ulp-iomuxc1", }, { /* sentinel */ } }; +MODULE_DEVICE_TABLE(of, imx8ulp_pinctrl_of_match); static int imx8ulp_pinctrl_probe(struct platform_device *pdev) { diff --git a/drivers/pinctrl/mediatek/pinctrl-mt6765.c b/drivers/pinctrl/mediatek/pinctrl-mt6765.c index f6ec41eb6e0c..72609cf74760 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mt6765.c +++ b/drivers/pinctrl/mediatek/pinctrl-mt6765.c @@ -1086,6 +1086,7 @@ static const struct of_device_id mt6765_pinctrl_of_match[] = { { .compatible = "mediatek,mt6765-pinctrl", .data = &mt6765_data }, { } }; +MODULE_DEVICE_TABLE(of, mt6765_pinctrl_of_match); static struct platform_driver mt6765_pinctrl_driver = { .driver = { diff --git a/drivers/pinctrl/mediatek/pinctrl-mt6779.c b/drivers/pinctrl/mediatek/pinctrl-mt6779.c index 62d4f5ad6737..591905e4132a 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mt6779.c +++ b/drivers/pinctrl/mediatek/pinctrl-mt6779.c @@ -762,6 +762,7 @@ static const struct of_device_id mt6779_pinctrl_of_match[] = { { .compatible = "mediatek,mt6779-pinctrl", .data = &mt6779_data }, { } }; +MODULE_DEVICE_TABLE(of, mt6779_pinctrl_of_match); static struct platform_driver mt6779_pinctrl_driver = { .driver = { diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c index c34719b7506d..4c4ada06423d 100644 --- a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c +++ b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c @@ -834,8 +834,6 @@ static int armada_37xx_gpiochip_register(struct platform_device *pdev, static int armada_37xx_add_function(struct armada_37xx_pmx_func *funcs, int *funcsize, const char *name) { - int i = 0; - if (*funcsize <= 0) return -EOVERFLOW; @@ -847,7 +845,6 @@ static int armada_37xx_add_function(struct armada_37xx_pmx_func *funcs, return -EEXIST; } funcs++; - i++; } /* append new unique function */ diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c index cada5d18ffae..80de389199bd 100644 --- a/drivers/pinctrl/pinconf-generic.c +++ b/drivers/pinctrl/pinconf-generic.c @@ -88,7 +88,7 @@ static void pinconf_generic_dump_one(struct pinctrl_dev *pctldev, seq_puts(s, items[i].display); /* Print unit if available */ if (items[i].has_arg) { - seq_printf(s, " (%u", + seq_printf(s, " (0x%x", pinconf_to_config_argument(config)); if (items[i].format) seq_printf(s, " %s)", items[i].format); diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index 4edd371c469f..b5e1c467625b 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -1,28 +1,29 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Awinic AW9523B i2c pin controller driver - * Copyright (c) 2020, AngeloGioacchino Del Regno - * <angelogioacchino.delregno@somainline.org> + * Copyright (c) 2020, AngeloGioacchino Del Regno <angelogioacchino.delregno@somainline.org> */ #include <linux/bitfield.h> +#include <linux/errno.h> #include <linux/gpio/consumer.h> #include <linux/gpio/driver.h> #include <linux/i2c.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/irq.h> -#include <linux/mutex.h> #include <linux/module.h> -#include <linux/pinctrl/pinconf.h> -#include <linux/pinctrl/pinctrl.h> -#include <linux/pinctrl/pinmux.h> -#include <linux/pinctrl/pinconf-generic.h> +#include <linux/mutex.h> #include <linux/property.h> #include <linux/regmap.h> #include <linux/regulator/consumer.h> #include <linux/slab.h> +#include <linux/pinctrl/pinconf-generic.h> +#include <linux/pinctrl/pinconf.h> +#include <linux/pinctrl/pinctrl.h> +#include <linux/pinctrl/pinmux.h> + #define AW9523_MAX_FUNCS 2 #define AW9523_NUM_PORTS 2 #define AW9523_PINS_PER_PORT 8 @@ -56,28 +57,14 @@ /* * struct aw9523_irq - Interrupt controller structure * @lock: mutex locking for the irq bus - * @irqchip: structure holding irqchip params * @cached_gpio: stores the previous gpio status for bit comparison */ struct aw9523_irq { struct mutex lock; - struct irq_chip *irqchip; u16 cached_gpio; }; /* - * struct aw9523_pinmux - Pin mux params - * @name: Name of the mux - * @grps: Groups of the mux - * @num_grps: Number of groups (sizeof array grps) - */ -struct aw9523_pinmux { - const char *name; - const char * const *grps; - const u8 num_grps; -}; - -/* * struct aw9523 - Main driver structure * @dev: device handle * @regmap: regmap handle for current device @@ -151,23 +138,16 @@ static const struct pinctrl_ops aw9523_pinctrl_ops = { }; static const char * const gpio_pwm_groups[] = { - "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", - "gpio6", "gpio7", "gpio8", "gpio9", "gpio10", "gpio11", - "gpio12", "gpio13", "gpio14", "gpio15" + "gpio0", "gpio1", "gpio2", "gpio3", /* 0-3 */ + "gpio4", "gpio5", "gpio6", "gpio7", /* 4-7 */ + "gpio8", "gpio9", "gpio10", "gpio11", /* 8-11 */ + "gpio12", "gpio13", "gpio14", "gpio15", /* 11-15 */ }; /* Warning: Do NOT reorder this array */ -static const struct aw9523_pinmux aw9523_pmx[] = { - { - .name = "pwm", - .grps = gpio_pwm_groups, - .num_grps = ARRAY_SIZE(gpio_pwm_groups), - }, - { - .name = "gpio", - .grps = gpio_pwm_groups, - .num_grps = ARRAY_SIZE(gpio_pwm_groups), - }, +static const struct pinfunction aw9523_pmx[] = { + PINCTRL_PINFUNCTION("pwm", gpio_pwm_groups, ARRAY_SIZE(gpio_pwm_groups)), + PINCTRL_PINFUNCTION("gpio", gpio_pwm_groups, ARRAY_SIZE(gpio_pwm_groups)), }; static int aw9523_pmx_get_funcs_count(struct pinctrl_dev *pctl) @@ -183,10 +163,10 @@ static const char *aw9523_pmx_get_fname(struct pinctrl_dev *pctl, static int aw9523_pmx_get_groups(struct pinctrl_dev *pctl, unsigned int sel, const char * const **groups, - unsigned int * const num_groups) + unsigned int * const ngroups) { - *groups = aw9523_pmx[sel].grps; - *num_groups = aw9523_pmx[sel].num_grps; + *groups = aw9523_pmx[sel].groups; + *ngroups = aw9523_pmx[sel].ngroups; return 0; } @@ -239,7 +219,7 @@ static int aw9523_pcfg_param_to_reg(enum pin_config_param pcp, int pin, u8 *r) reg = AW9523_REG_OUT_STATE(pin); break; default: - return -EOPNOTSUPP; + return -ENOTSUPP; } *r = reg; @@ -290,7 +270,7 @@ static int aw9523_pconf_get(struct pinctrl_dev *pctldev, unsigned int pin, val = FIELD_GET(AW9523_GCR_GPOMD_MASK, val); break; default: - return -EOPNOTSUPP; + return -ENOTSUPP; } if (val < 1) return -EINVAL; @@ -344,7 +324,7 @@ static int aw9523_pconf_set(struct pinctrl_dev *pctldev, unsigned int pin, case PIN_CONFIG_DRIVE_OPEN_DRAIN: /* Open-Drain is supported only on port 0 */ if (pin >= AW9523_PINS_PER_PORT) { - rc = -EOPNOTSUPP; + rc = -ENOTSUPP; goto end; } mask = AW9523_GCR_GPOMD_MASK; @@ -361,7 +341,7 @@ static int aw9523_pconf_set(struct pinctrl_dev *pctldev, unsigned int pin, val = AW9523_GCR_GPOMD_MASK; break; default: - rc = -EOPNOTSUPP; + rc = -ENOTSUPP; goto end; } @@ -408,8 +388,8 @@ static int aw9523_get_pin_direction(struct regmap *regmap, u8 pin, u8 n) * * Return: Zero for success or negative number for error */ -static int aw9523_get_port_state(struct regmap *regmap, u8 pin, - u8 regbit, unsigned int *state) +static int aw9523_get_port_state(struct regmap *regmap, u8 pin, u8 regbit, + unsigned int *state) { u8 reg; int dir; @@ -447,12 +427,12 @@ static int aw9523_gpio_irq_type(struct irq_data *d, unsigned int type) static void aw9523_irq_mask(struct irq_data *d) { struct aw9523 *awi = gpiochip_get_data(irq_data_get_irq_chip_data(d)); - unsigned int n = d->hwirq % AW9523_PINS_PER_PORT; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned int n = hwirq % AW9523_PINS_PER_PORT; - regmap_update_bits(awi->regmap, - AW9523_REG_INTR_DIS(d->hwirq), + regmap_update_bits(awi->regmap, AW9523_REG_INTR_DIS(hwirq), BIT(n), BIT(n)); - gpiochip_disable_irq(&awi->gpio, irqd_to_hwirq(d)); + gpiochip_disable_irq(&awi->gpio, hwirq); } /* @@ -465,11 +445,11 @@ static void aw9523_irq_mask(struct irq_data *d) static void aw9523_irq_unmask(struct irq_data *d) { struct aw9523 *awi = gpiochip_get_data(irq_data_get_irq_chip_data(d)); - unsigned int n = d->hwirq % AW9523_PINS_PER_PORT; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned int n = hwirq % AW9523_PINS_PER_PORT; - gpiochip_enable_irq(&awi->gpio, irqd_to_hwirq(d)); - regmap_update_bits(awi->regmap, - AW9523_REG_INTR_DIS(d->hwirq), + gpiochip_enable_irq(&awi->gpio, hwirq); + regmap_update_bits(awi->regmap, AW9523_REG_INTR_DIS(hwirq), BIT(n), 0); } @@ -622,7 +602,7 @@ static int aw9523_gpio_get_multiple(struct gpio_chip *chip, mutex_lock(&awi->i2c_lock); /* Port 0 (gpio 0-7) */ - m = *mask & U8_MAX; + m = *mask; if (m) { ret = _aw9523_gpio_get_multiple(awi, 0, &state, m); if (ret) @@ -631,7 +611,7 @@ static int aw9523_gpio_get_multiple(struct gpio_chip *chip, *bits = state; /* Port 1 (gpio 8-15) */ - m = (*mask >> 8) & U8_MAX; + m = *mask >> 8; if (m) { ret = _aw9523_gpio_get_multiple(awi, AW9523_PINS_PER_PORT, &state, m); @@ -652,29 +632,26 @@ static void aw9523_gpio_set_multiple(struct gpio_chip *chip, struct aw9523 *awi = gpiochip_get_data(chip); u8 mask_lo, mask_hi, bits_lo, bits_hi; unsigned int reg; - int ret = 0; + int ret; + + mask_lo = *mask; + mask_hi = *mask >> 8; + bits_lo = *bits; + bits_hi = *bits >> 8; - mask_lo = *mask & U8_MAX; - mask_hi = (*mask >> 8) & U8_MAX; mutex_lock(&awi->i2c_lock); if (mask_hi) { reg = AW9523_REG_OUT_STATE(AW9523_PINS_PER_PORT); - bits_hi = (*bits >> 8) & U8_MAX; - ret = regmap_write_bits(awi->regmap, reg, mask_hi, bits_hi); - if (ret) { + if (ret) dev_warn(awi->dev, "Cannot write port1 out level\n"); - goto out; - } } if (mask_lo) { reg = AW9523_REG_OUT_STATE(0); - bits_lo = *bits & U8_MAX; ret = regmap_write_bits(awi->regmap, reg, mask_lo, bits_lo); if (ret) dev_warn(awi->dev, "Cannot write port0 out level\n"); } -out: mutex_unlock(&awi->i2c_lock); } @@ -827,29 +804,21 @@ static int aw9523_init_irq(struct aw9523 *awi, int irq) { struct device *dev = awi->dev; struct gpio_irq_chip *girq; - struct irq_chip *irqchip; int ret; if (!device_property_read_bool(dev, "interrupt-controller")) return 0; - irqchip = devm_kzalloc(dev, sizeof(*irqchip), GFP_KERNEL); - if (!irqchip) - return -ENOMEM; - awi->irq = devm_kzalloc(dev, sizeof(*awi->irq), GFP_KERNEL); if (!awi->irq) return -ENOMEM; - awi->irq->irqchip = irqchip; mutex_init(&awi->irq->lock); ret = devm_request_threaded_irq(dev, irq, NULL, aw9523_irq_thread_func, IRQF_ONESHOT, dev_name(dev), awi); - if (ret) { - dev_err(dev, "Failed to request irq %d\n", irq); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Failed to request irq %d\n", irq); girq = &awi->gpio.irq; gpio_irq_chip_set_chip(girq, &aw9523_irq_chip); @@ -1015,8 +984,7 @@ static int aw9523_probe(struct i2c_client *client) } mutex_init(&awi->i2c_lock); - lockdep_set_subclass(&awi->i2c_lock, - i2c_adapter_depth(client->adapter)); + lockdep_set_subclass(&awi->i2c_lock, i2c_adapter_depth(client->adapter)); pdesc = devm_kzalloc(dev, sizeof(*pdesc), GFP_KERNEL); if (!pdesc) @@ -1046,8 +1014,7 @@ static int aw9523_probe(struct i2c_client *client) awi->pctl = devm_pinctrl_register(dev, pdesc, awi); if (IS_ERR(awi->pctl)) { - ret = PTR_ERR(awi->pctl); - dev_err(dev, "Cannot register pinctrl: %d", ret); + ret = dev_err_probe(dev, PTR_ERR(awi->pctl), "Cannot register pinctrl"); goto err_disable_vregs; } @@ -1067,10 +1034,6 @@ err_disable_vregs: static void aw9523_remove(struct i2c_client *client) { struct aw9523 *awi = i2c_get_clientdata(client); - int ret; - - if (!awi) - return; /* * If the chip VIO is connected to a regulator that we can turn @@ -1082,10 +1045,8 @@ static void aw9523_remove(struct i2c_client *client) regulator_disable(awi->vio_vreg); } else { mutex_lock(&awi->i2c_lock); - ret = aw9523_hw_init(awi); + aw9523_hw_init(awi); mutex_unlock(&awi->i2c_lock); - if (ret) - return; } mutex_destroy(&awi->i2c_lock); diff --git a/drivers/pinctrl/pinctrl-loongson2.c b/drivers/pinctrl/pinctrl-loongson2.c index a72ffeca26fb..4d4fbeadafb7 100644 --- a/drivers/pinctrl/pinctrl-loongson2.c +++ b/drivers/pinctrl/pinctrl-loongson2.c @@ -286,6 +286,7 @@ static const struct of_device_id loongson2_pinctrl_dt_match[] = { }, { } }; +MODULE_DEVICE_TABLE(of, loongson2_pinctrl_dt_match); static struct platform_driver loongson2_pinctrl_driver = { .probe = loongson2_pinctrl_probe, diff --git a/drivers/pinctrl/pinctrl-max77620.c b/drivers/pinctrl/pinctrl-max77620.c index ab723ab4ec1d..d236daa7c13e 100644 --- a/drivers/pinctrl/pinctrl-max77620.c +++ b/drivers/pinctrl/pinctrl-max77620.c @@ -88,7 +88,6 @@ struct max77620_pingroup { struct max77620_pin_info { enum max77620_pin_ppdrv drv_type; - int pull_config; }; struct max77620_fps_config { @@ -104,7 +103,6 @@ struct max77620_pctrl_info { struct device *dev; struct pinctrl_dev *pctl; struct regmap *rmap; - int pins_current_opt[MAX77620_GPIO_NR]; const struct max77620_pin_function *functions; unsigned int num_functions; const struct max77620_pingroup *pin_groups; diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 19cc0db771a5..a798f31d6954 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -81,8 +81,6 @@ struct pcs_conf_type { * @name: pinctrl function name * @vals: register and vals array * @nvals: number of entries in vals array - * @pgnames: array of pingroup names the function uses - * @npgnames: number of pingroup names the function uses * @conf: array of pin configurations * @nconfs: number of pin configurations available * @node: list node @@ -91,8 +89,6 @@ struct pcs_function { const char *name; struct pcs_func_vals *vals; unsigned nvals; - const char **pgnames; - int npgnames; struct pcs_conf_vals *conf; int nconfs; struct list_head node; @@ -554,21 +550,30 @@ static int pcs_pinconf_set(struct pinctrl_dev *pctldev, unsigned offset = 0, shift = 0, i, data, ret; u32 arg; int j; + enum pin_config_param param; ret = pcs_get_function(pctldev, pin, &func); if (ret) return ret; for (j = 0; j < num_configs; j++) { + param = pinconf_to_config_param(configs[j]); + + /* BIAS_DISABLE has no entry in the func->conf table */ + if (param == PIN_CONFIG_BIAS_DISABLE) { + /* This just disables all bias entries */ + pcs_pinconf_clear_bias(pctldev, pin); + continue; + } + for (i = 0; i < func->nconfs; i++) { - if (pinconf_to_config_param(configs[j]) - != func->conf[i].param) + if (param != func->conf[i].param) continue; offset = pin * (pcs->width / BITS_PER_BYTE); data = pcs->read(pcs->base + offset); arg = pinconf_to_config_argument(configs[j]); - switch (func->conf[i].param) { + switch (param) { /* 2 parameters */ case PIN_CONFIG_INPUT_SCHMITT: case PIN_CONFIG_DRIVE_STRENGTH: @@ -580,9 +585,6 @@ static int pcs_pinconf_set(struct pinctrl_dev *pctldev, data |= (arg << shift) & func->conf[i].mask; break; /* 4 parameters */ - case PIN_CONFIG_BIAS_DISABLE: - pcs_pinconf_clear_bias(pctldev, pin); - break; case PIN_CONFIG_BIAS_PULL_DOWN: case PIN_CONFIG_BIAS_PULL_UP: if (arg) @@ -1625,7 +1627,6 @@ static int pcs_irq_init_chained_handler(struct pcs_device *pcs, return 0; } -#ifdef CONFIG_PM static int pcs_save_context(struct pcs_device *pcs) { int i, mux_bytes; @@ -1690,14 +1691,9 @@ static void pcs_restore_context(struct pcs_device *pcs) } } -static int pinctrl_single_suspend(struct platform_device *pdev, - pm_message_t state) +static int pinctrl_single_suspend_noirq(struct device *dev) { - struct pcs_device *pcs; - - pcs = platform_get_drvdata(pdev); - if (!pcs) - return -EINVAL; + struct pcs_device *pcs = dev_get_drvdata(dev); if (pcs->flags & PCS_CONTEXT_LOSS_OFF) { int ret; @@ -1710,20 +1706,19 @@ static int pinctrl_single_suspend(struct platform_device *pdev, return pinctrl_force_sleep(pcs->pctl); } -static int pinctrl_single_resume(struct platform_device *pdev) +static int pinctrl_single_resume_noirq(struct device *dev) { - struct pcs_device *pcs; - - pcs = platform_get_drvdata(pdev); - if (!pcs) - return -EINVAL; + struct pcs_device *pcs = dev_get_drvdata(dev); if (pcs->flags & PCS_CONTEXT_LOSS_OFF) pcs_restore_context(pcs); return pinctrl_force_default(pcs->pctl); } -#endif + +static DEFINE_NOIRQ_DEV_PM_OPS(pinctrl_single_pm_ops, + pinctrl_single_suspend_noirq, + pinctrl_single_resume_noirq); /** * pcs_quirk_missing_pinctrl_cells - handle legacy binding @@ -1986,11 +1981,8 @@ static struct platform_driver pcs_driver = { .driver = { .name = DRIVER_NAME, .of_match_table = pcs_of_match, + .pm = pm_sleep_ptr(&pinctrl_single_pm_ops), }, -#ifdef CONFIG_PM - .suspend = pinctrl_single_suspend, - .resume = pinctrl_single_resume, -#endif }; module_platform_driver(pcs_driver); diff --git a/drivers/pinctrl/pinmux.c b/drivers/pinctrl/pinmux.c index d924207d629b..addba55334d9 100644 --- a/drivers/pinctrl/pinmux.c +++ b/drivers/pinctrl/pinmux.c @@ -578,6 +578,7 @@ static int pinmux_functions_show(struct seq_file *s, void *what) return 0; } +DEFINE_SHOW_ATTRIBUTE(pinmux_functions); static int pinmux_pins_show(struct seq_file *s, void *what) { @@ -650,6 +651,7 @@ static int pinmux_pins_show(struct seq_file *s, void *what) return 0; } +DEFINE_SHOW_ATTRIBUTE(pinmux_pins); void pinmux_show_map(struct seq_file *s, const struct pinctrl_map *map) { @@ -672,10 +674,12 @@ void pinmux_show_setting(struct seq_file *s, setting->data.mux.func); } -DEFINE_SHOW_ATTRIBUTE(pinmux_functions); -DEFINE_SHOW_ATTRIBUTE(pinmux_pins); +static int pinmux_select_show(struct seq_file *s, void *unused) +{ + return -EPERM; +} -static ssize_t pinmux_select(struct file *file, const char __user *user_buf, +static ssize_t pinmux_select_write(struct file *file, const char __user *user_buf, size_t len, loff_t *ppos) { struct seq_file *sfile = file->private_data; @@ -749,19 +753,7 @@ exit_free_buf: return ret; } - -static int pinmux_select_open(struct inode *inode, struct file *file) -{ - return single_open(file, NULL, inode->i_private); -} - -static const struct file_operations pinmux_select_ops = { - .owner = THIS_MODULE, - .open = pinmux_select_open, - .write = pinmux_select, - .llseek = no_llseek, - .release = single_release, -}; +DEFINE_SHOW_STORE_ATTRIBUTE(pinmux_select); void pinmux_init_device_debugfs(struct dentry *devroot, struct pinctrl_dev *pctldev) @@ -771,7 +763,7 @@ void pinmux_init_device_debugfs(struct dentry *devroot, debugfs_create_file("pinmux-pins", 0444, devroot, pctldev, &pinmux_pins_fops); debugfs_create_file("pinmux-select", 0200, - devroot, pctldev, &pinmux_select_ops); + devroot, pctldev, &pinmux_select_fops); } #endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/pinctrl/pxa/pinctrl-pxa2xx.c b/drivers/pinctrl/pxa/pinctrl-pxa2xx.c index d2568dab8c78..9e34b92ff5f2 100644 --- a/drivers/pinctrl/pxa/pinctrl-pxa2xx.c +++ b/drivers/pinctrl/pxa/pinctrl-pxa2xx.c @@ -32,7 +32,7 @@ static const char *pxa2xx_pctrl_get_group_name(struct pinctrl_dev *pctldev, unsigned tgroup) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_group *group = pctl->groups + tgroup; + struct pingroup *group = pctl->groups + tgroup; return group->name; } @@ -43,10 +43,10 @@ static int pxa2xx_pctrl_get_group_pins(struct pinctrl_dev *pctldev, unsigned *num_pins) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_group *group = pctl->groups + tgroup; + struct pingroup *group = pctl->groups + tgroup; - *pins = (unsigned *)&group->pin; - *num_pins = 1; + *pins = group->pins; + *num_pins = group->npins; return 0; } @@ -109,7 +109,7 @@ static const char *pxa2xx_pmx_get_func_name(struct pinctrl_dev *pctldev, unsigned function) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_function *pf = pctl->functions + function; + struct pinfunction *pf = pctl->functions + function; return pf->name; } @@ -127,7 +127,7 @@ static int pxa2xx_pmx_get_func_groups(struct pinctrl_dev *pctldev, unsigned * const num_groups) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_function *pf = pctl->functions + function; + struct pinfunction *pf = pctl->functions + function; *groups = pf->groups; *num_groups = pf->ngroups; @@ -139,20 +139,18 @@ static int pxa2xx_pmx_set_mux(struct pinctrl_dev *pctldev, unsigned function, unsigned tgroup) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_group *group = pctl->groups + tgroup; + struct pingroup *g = pctl->groups + tgroup; + unsigned int pin = g->pins[0]; struct pxa_desc_function *df; - int pin, shift; unsigned long flags; void __iomem *gafr, *gpdr; + int shift; u32 val; - - df = pxa_desc_by_func_group(pctl, group->name, - (pctl->functions + function)->name); + df = pxa_desc_by_func_group(pctl, g->name, (pctl->functions + function)->name); if (!df) return -EINVAL; - pin = group->pin; gafr = pctl->base_gafr[pin / 16]; gpdr = pctl->base_gpdr[pin / 32]; shift = (pin % 16) << 1; @@ -186,9 +184,9 @@ static int pxa2xx_pconf_group_get(struct pinctrl_dev *pctldev, unsigned long *config) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_group *g = pctl->groups + group; + struct pingroup *g = pctl->groups + group; + unsigned int pin = g->pins[0]; unsigned long flags; - unsigned pin = g->pin; void __iomem *pgsr = pctl->base_pgsr[pin / 32]; u32 val; @@ -208,9 +206,9 @@ static int pxa2xx_pconf_group_set(struct pinctrl_dev *pctldev, unsigned num_configs) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_group *g = pctl->groups + group; + struct pingroup *g = pctl->groups + group; + unsigned int pin = g->pins[0]; unsigned long flags; - unsigned pin = g->pin; void __iomem *pgsr = pctl->base_pgsr[pin / 32]; int i, is_set = 0; u32 val; @@ -249,11 +247,11 @@ static struct pinctrl_desc pxa2xx_pinctrl_desc = { .pmxops = &pxa2xx_pinmux_ops, }; -static const struct pxa_pinctrl_function * -pxa2xx_find_function(struct pxa_pinctrl *pctl, const char *fname, - const struct pxa_pinctrl_function *functions) +static const struct pinfunction *pxa2xx_find_function(struct pxa_pinctrl *pctl, + const char *fname, + const struct pinfunction *functions) { - const struct pxa_pinctrl_function *func; + const struct pinfunction *func; for (func = functions; func->name; func++) if (!strcmp(fname, func->name)) @@ -264,8 +262,8 @@ pxa2xx_find_function(struct pxa_pinctrl *pctl, const char *fname, static int pxa2xx_build_functions(struct pxa_pinctrl *pctl) { + struct pinfunction *functions; int i; - struct pxa_pinctrl_function *functions; struct pxa_desc_function *df; /* @@ -296,9 +294,9 @@ static int pxa2xx_build_functions(struct pxa_pinctrl *pctl) static int pxa2xx_build_groups(struct pxa_pinctrl *pctl) { int i, j, ngroups; - struct pxa_pinctrl_function *func; struct pxa_desc_function *df; - char **gtmp; + struct pinfunction *func; + const char **gtmp; gtmp = devm_kmalloc_array(pctl->dev, pctl->npins, sizeof(*gtmp), GFP_KERNEL); @@ -316,13 +314,9 @@ static int pxa2xx_build_groups(struct pxa_pinctrl *pctl) pctl->ppins[j].pin.name; func = pctl->functions + i; func->ngroups = ngroups; - func->groups = - devm_kmalloc_array(pctl->dev, ngroups, - sizeof(char *), GFP_KERNEL); + func->groups = devm_kmemdup(pctl->dev, gtmp, ngroups * sizeof(*gtmp), GFP_KERNEL); if (!func->groups) return -ENOMEM; - - memcpy(func->groups, gtmp, ngroups * sizeof(*gtmp)); } devm_kfree(pctl->dev, gtmp); @@ -332,8 +326,8 @@ static int pxa2xx_build_groups(struct pxa_pinctrl *pctl) static int pxa2xx_build_state(struct pxa_pinctrl *pctl, const struct pxa_desc_pin *ppins, int npins) { - struct pxa_pinctrl_group *group; struct pinctrl_pin_desc *pins; + struct pingroup *group; int ret, i; pctl->npins = npins; @@ -357,7 +351,8 @@ static int pxa2xx_build_state(struct pxa_pinctrl *pctl, for (i = 0; i < npins; i++) { group = pctl->groups + i; group->name = ppins[i].pin.name; - group->pin = ppins[i].pin.number; + group->pins = &ppins[i].pin.number; + group->npins = 1; } ret = pxa2xx_build_functions(pctl); diff --git a/drivers/pinctrl/pxa/pinctrl-pxa2xx.h b/drivers/pinctrl/pxa/pinctrl-pxa2xx.h index d86d47dbbc94..b292b79efdf8 100644 --- a/drivers/pinctrl/pxa/pinctrl-pxa2xx.h +++ b/drivers/pinctrl/pxa/pinctrl-pxa2xx.h @@ -52,17 +52,6 @@ struct pxa_desc_pin { struct pxa_desc_function *functions; }; -struct pxa_pinctrl_group { - const char *name; - unsigned pin; -}; - -struct pxa_pinctrl_function { - const char *name; - const char **groups; - unsigned ngroups; -}; - struct pxa_pinctrl { spinlock_t lock; void __iomem **base_gafr; @@ -74,9 +63,9 @@ struct pxa_pinctrl { unsigned npins; const struct pxa_desc_pin *ppins; unsigned ngroups; - struct pxa_pinctrl_group *groups; + struct pingroup *groups; unsigned nfuncs; - struct pxa_pinctrl_function *functions; + struct pinfunction *functions; char *name; }; diff --git a/drivers/pinctrl/qcom/pinctrl-sm7150.c b/drivers/pinctrl/qcom/pinctrl-sm7150.c index c25357ca1963..095a1ca75849 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm7150.c +++ b/drivers/pinctrl/qcom/pinctrl-sm7150.c @@ -65,7 +65,7 @@ enum { .intr_detection_width = 2, \ } -#define SDC_QDSD_PINGROUP(pg_name, ctl, pull, drv) \ +#define SDC_QDSD_PINGROUP(pg_name, _tile, ctl, pull, drv) \ { \ .grp = PINCTRL_PINGROUP(#pg_name, \ pg_name##_pins, \ @@ -75,7 +75,7 @@ enum { .intr_cfg_reg = 0, \ .intr_status_reg = 0, \ .intr_target_reg = 0, \ - .tile = SOUTH, \ + .tile = _tile, \ .mux_bit = -1, \ .pull_bit = pull, \ .drv_bit = drv, \ @@ -101,7 +101,7 @@ enum { .intr_cfg_reg = 0, \ .intr_status_reg = 0, \ .intr_target_reg = 0, \ - .tile = SOUTH, \ + .tile = WEST, \ .mux_bit = -1, \ .pull_bit = 3, \ .drv_bit = 0, \ @@ -1199,13 +1199,13 @@ static const struct msm_pingroup sm7150_groups[] = { [117] = PINGROUP(117, NORTH, _, _, _, _, _, _, _, _, _), [118] = PINGROUP(118, NORTH, _, _, _, _, _, _, _, _, _), [119] = UFS_RESET(ufs_reset, 0x9f000), - [120] = SDC_QDSD_PINGROUP(sdc1_rclk, 0x9a000, 15, 0), - [121] = SDC_QDSD_PINGROUP(sdc1_clk, 0x9a000, 13, 6), - [122] = SDC_QDSD_PINGROUP(sdc1_cmd, 0x9a000, 11, 3), - [123] = SDC_QDSD_PINGROUP(sdc1_data, 0x9a000, 9, 0), - [124] = SDC_QDSD_PINGROUP(sdc2_clk, 0x98000, 14, 6), - [125] = SDC_QDSD_PINGROUP(sdc2_cmd, 0x98000, 11, 3), - [126] = SDC_QDSD_PINGROUP(sdc2_data, 0x98000, 9, 0), + [120] = SDC_QDSD_PINGROUP(sdc1_rclk, WEST, 0x9a000, 15, 0), + [121] = SDC_QDSD_PINGROUP(sdc1_clk, WEST, 0x9a000, 13, 6), + [122] = SDC_QDSD_PINGROUP(sdc1_cmd, WEST, 0x9a000, 11, 3), + [123] = SDC_QDSD_PINGROUP(sdc1_data, WEST, 0x9a000, 9, 0), + [124] = SDC_QDSD_PINGROUP(sdc2_clk, SOUTH, 0x98000, 14, 6), + [125] = SDC_QDSD_PINGROUP(sdc2_cmd, SOUTH, 0x98000, 11, 3), + [126] = SDC_QDSD_PINGROUP(sdc2_data, SOUTH, 0x98000, 9, 0), }; static const struct msm_gpio_wakeirq_map sm7150_pdc_map[] = { @@ -1246,6 +1246,7 @@ static const struct of_device_id sm7150_tlmm_of_match[] = { { .compatible = "qcom,sm7150-tlmm", }, { }, }; +MODULE_DEVICE_TABLE(of, sm7150_tlmm_of_match); static struct platform_driver sm7150_tlmm_driver = { .driver = { diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c index f4e2c88a7c82..4e80c7204e5f 100644 --- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c +++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c @@ -1202,6 +1202,7 @@ static const struct of_device_id pmic_gpio_of_match[] = { { .compatible = "qcom,pm6150-gpio", .data = (void *) 10 }, { .compatible = "qcom,pm6150l-gpio", .data = (void *) 12 }, { .compatible = "qcom,pm6350-gpio", .data = (void *) 9 }, + { .compatible = "qcom,pm6450-gpio", .data = (void *) 9 }, { .compatible = "qcom,pm7250b-gpio", .data = (void *) 12 }, { .compatible = "qcom,pm7325-gpio", .data = (void *) 10 }, { .compatible = "qcom,pm7550ba-gpio", .data = (void *) 8}, @@ -1234,10 +1235,12 @@ static const struct of_device_id pmic_gpio_of_match[] = { { .compatible = "qcom,pm8994-gpio", .data = (void *) 22 }, { .compatible = "qcom,pm8998-gpio", .data = (void *) 26 }, { .compatible = "qcom,pma8084-gpio", .data = (void *) 22 }, + { .compatible = "qcom,pmd8028-gpio", .data = (void *) 4 }, { .compatible = "qcom,pmi632-gpio", .data = (void *) 8 }, { .compatible = "qcom,pmi8950-gpio", .data = (void *) 2 }, { .compatible = "qcom,pmi8994-gpio", .data = (void *) 10 }, { .compatible = "qcom,pmi8998-gpio", .data = (void *) 14 }, + { .compatible = "qcom,pmih0108-gpio", .data = (void *) 18 }, { .compatible = "qcom,pmk8350-gpio", .data = (void *) 4 }, { .compatible = "qcom,pmk8550-gpio", .data = (void *) 6 }, { .compatible = "qcom,pmm8155au-gpio", .data = (void *) 10 }, @@ -1253,6 +1256,7 @@ static const struct of_device_id pmic_gpio_of_match[] = { { .compatible = "qcom,pmx55-gpio", .data = (void *) 11 }, { .compatible = "qcom,pmx65-gpio", .data = (void *) 16 }, { .compatible = "qcom,pmx75-gpio", .data = (void *) 16 }, + { .compatible = "qcom,pmxr2230-gpio", .data = (void *) 12 }, { }, }; diff --git a/drivers/pinctrl/realtek/pinctrl-rtd1315e.c b/drivers/pinctrl/realtek/pinctrl-rtd1315e.c index 10afc736a52b..86f919122bed 100644 --- a/drivers/pinctrl/realtek/pinctrl-rtd1315e.c +++ b/drivers/pinctrl/realtek/pinctrl-rtd1315e.c @@ -1414,6 +1414,7 @@ static const struct of_device_id rtd1315e_pinctrl_of_match[] = { { .compatible = "realtek,rtd1315e-pinctrl", }, {}, }; +MODULE_DEVICE_TABLE(of, rtd1315e_pinctrl_of_match); static struct platform_driver rtd1315e_pinctrl_driver = { .driver = { diff --git a/drivers/pinctrl/realtek/pinctrl-rtd1319d.c b/drivers/pinctrl/realtek/pinctrl-rtd1319d.c index b1a654ac30dc..474c337d2d05 100644 --- a/drivers/pinctrl/realtek/pinctrl-rtd1319d.c +++ b/drivers/pinctrl/realtek/pinctrl-rtd1319d.c @@ -1584,6 +1584,7 @@ static const struct of_device_id rtd1319d_pinctrl_of_match[] = { { .compatible = "realtek,rtd1319d-pinctrl", }, {}, }; +MODULE_DEVICE_TABLE(of, rtd1319d_pinctrl_of_match); static struct platform_driver rtd1319d_pinctrl_driver = { .driver = { diff --git a/drivers/pinctrl/renesas/pfc-r8a779h0.c b/drivers/pinctrl/renesas/pfc-r8a779h0.c index afa8f06c85cf..438d1f2739dd 100644 --- a/drivers/pinctrl/renesas/pfc-r8a779h0.c +++ b/drivers/pinctrl/renesas/pfc-r8a779h0.c @@ -75,10 +75,10 @@ #define GPSR0_9 F_(MSIOF5_SYNC, IP1SR0_7_4) #define GPSR0_8 F_(MSIOF5_SS1, IP1SR0_3_0) #define GPSR0_7 F_(MSIOF5_SS2, IP0SR0_31_28) -#define GPSR0_6 F_(IRQ0, IP0SR0_27_24) -#define GPSR0_5 F_(IRQ1, IP0SR0_23_20) -#define GPSR0_4 F_(IRQ2, IP0SR0_19_16) -#define GPSR0_3 F_(IRQ3, IP0SR0_15_12) +#define GPSR0_6 F_(IRQ0_A, IP0SR0_27_24) +#define GPSR0_5 F_(IRQ1_A, IP0SR0_23_20) +#define GPSR0_4 F_(IRQ2_A, IP0SR0_19_16) +#define GPSR0_3 F_(IRQ3_A, IP0SR0_15_12) #define GPSR0_2 F_(GP0_02, IP0SR0_11_8) #define GPSR0_1 F_(GP0_01, IP0SR0_7_4) #define GPSR0_0 F_(GP0_00, IP0SR0_3_0) @@ -265,10 +265,10 @@ #define IP0SR0_3_0 F_(0, 0) FM(ERROROUTC_N_B) FM(TCLK2_B) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) #define IP0SR0_7_4 F_(0, 0) FM(MSIOF3_SS1) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) #define IP0SR0_11_8 F_(0, 0) FM(MSIOF3_SS2) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) -#define IP0SR0_15_12 FM(IRQ3) FM(MSIOF3_SCK) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) -#define IP0SR0_19_16 FM(IRQ2) FM(MSIOF3_TXD) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) -#define IP0SR0_23_20 FM(IRQ1) FM(MSIOF3_RXD) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) -#define IP0SR0_27_24 FM(IRQ0) FM(MSIOF3_SYNC) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) +#define IP0SR0_15_12 FM(IRQ3_A) FM(MSIOF3_SCK) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) +#define IP0SR0_19_16 FM(IRQ2_A) FM(MSIOF3_TXD) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) +#define IP0SR0_23_20 FM(IRQ1_A) FM(MSIOF3_RXD) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) +#define IP0SR0_27_24 FM(IRQ0_A) FM(MSIOF3_SYNC) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) #define IP0SR0_31_28 FM(MSIOF5_SS2) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) /* IP1SR0 */ /* 0 */ /* 1 */ /* 2 */ /* 3 4 5 6 7 8 9 A B C D E F */ @@ -672,16 +672,16 @@ static const u16 pinmux_data[] = { PINMUX_IPSR_GPSR(IP0SR0_11_8, MSIOF3_SS2), - PINMUX_IPSR_GPSR(IP0SR0_15_12, IRQ3), + PINMUX_IPSR_GPSR(IP0SR0_15_12, IRQ3_A), PINMUX_IPSR_GPSR(IP0SR0_15_12, MSIOF3_SCK), - PINMUX_IPSR_GPSR(IP0SR0_19_16, IRQ2), + PINMUX_IPSR_GPSR(IP0SR0_19_16, IRQ2_A), PINMUX_IPSR_GPSR(IP0SR0_19_16, MSIOF3_TXD), - PINMUX_IPSR_GPSR(IP0SR0_23_20, IRQ1), + PINMUX_IPSR_GPSR(IP0SR0_23_20, IRQ1_A), PINMUX_IPSR_GPSR(IP0SR0_23_20, MSIOF3_RXD), - PINMUX_IPSR_GPSR(IP0SR0_27_24, IRQ0), + PINMUX_IPSR_GPSR(IP0SR0_27_24, IRQ0_A), PINMUX_IPSR_GPSR(IP0SR0_27_24, MSIOF3_SYNC), PINMUX_IPSR_GPSR(IP0SR0_31_28, MSIOF5_SS2), @@ -1660,6 +1660,90 @@ static const unsigned int i2c3_mux[] = { SDA3_MARK, SCL3_MARK, }; +/* - INTC-EX ---------------------------------------------------------------- */ +static const unsigned int intc_ex_irq0_a_pins[] = { + /* IRQ0_A */ + RCAR_GP_PIN(0, 6), +}; +static const unsigned int intc_ex_irq0_a_mux[] = { + IRQ0_A_MARK, +}; +static const unsigned int intc_ex_irq0_b_pins[] = { + /* IRQ0_B */ + RCAR_GP_PIN(1, 20), +}; +static const unsigned int intc_ex_irq0_b_mux[] = { + IRQ0_B_MARK, +}; + +static const unsigned int intc_ex_irq1_a_pins[] = { + /* IRQ1_A */ + RCAR_GP_PIN(0, 5), +}; +static const unsigned int intc_ex_irq1_a_mux[] = { + IRQ1_A_MARK, +}; +static const unsigned int intc_ex_irq1_b_pins[] = { + /* IRQ1_B */ + RCAR_GP_PIN(1, 21), +}; +static const unsigned int intc_ex_irq1_b_mux[] = { + IRQ1_B_MARK, +}; + +static const unsigned int intc_ex_irq2_a_pins[] = { + /* IRQ2_A */ + RCAR_GP_PIN(0, 4), +}; +static const unsigned int intc_ex_irq2_a_mux[] = { + IRQ2_A_MARK, +}; +static const unsigned int intc_ex_irq2_b_pins[] = { + /* IRQ2_B */ + RCAR_GP_PIN(0, 13), +}; +static const unsigned int intc_ex_irq2_b_mux[] = { + IRQ2_B_MARK, +}; + +static const unsigned int intc_ex_irq3_a_pins[] = { + /* IRQ3_A */ + RCAR_GP_PIN(0, 3), +}; +static const unsigned int intc_ex_irq3_a_mux[] = { + IRQ3_A_MARK, +}; +static const unsigned int intc_ex_irq3_b_pins[] = { + /* IRQ3_B */ + RCAR_GP_PIN(1, 23), +}; +static const unsigned int intc_ex_irq3_b_mux[] = { + IRQ3_B_MARK, +}; + +static const unsigned int intc_ex_irq4_a_pins[] = { + /* IRQ4_A */ + RCAR_GP_PIN(1, 17), +}; +static const unsigned int intc_ex_irq4_a_mux[] = { + IRQ4_A_MARK, +}; +static const unsigned int intc_ex_irq4_b_pins[] = { + /* IRQ4_B */ + RCAR_GP_PIN(2, 3), +}; +static const unsigned int intc_ex_irq4_b_mux[] = { + IRQ4_B_MARK, +}; + +static const unsigned int intc_ex_irq5_pins[] = { + /* IRQ5 */ + RCAR_GP_PIN(2, 2), +}; +static const unsigned int intc_ex_irq5_mux[] = { + IRQ5_MARK, +}; + /* - MMC -------------------------------------------------------------------- */ static const unsigned int mmc_data_pins[] = { /* MMC_SD_D[0:3], MMC_D[4:7] */ @@ -2416,6 +2500,18 @@ static const struct sh_pfc_pin_group pinmux_groups[] = { SH_PFC_PIN_GROUP(i2c2), SH_PFC_PIN_GROUP(i2c3), + SH_PFC_PIN_GROUP(intc_ex_irq0_a), + SH_PFC_PIN_GROUP(intc_ex_irq0_b), + SH_PFC_PIN_GROUP(intc_ex_irq1_a), + SH_PFC_PIN_GROUP(intc_ex_irq1_b), + SH_PFC_PIN_GROUP(intc_ex_irq2_a), + SH_PFC_PIN_GROUP(intc_ex_irq2_b), + SH_PFC_PIN_GROUP(intc_ex_irq3_a), + SH_PFC_PIN_GROUP(intc_ex_irq3_b), + SH_PFC_PIN_GROUP(intc_ex_irq4_a), + SH_PFC_PIN_GROUP(intc_ex_irq4_b), + SH_PFC_PIN_GROUP(intc_ex_irq5), + BUS_DATA_PIN_GROUP(mmc_data, 1), BUS_DATA_PIN_GROUP(mmc_data, 4), BUS_DATA_PIN_GROUP(mmc_data, 8), @@ -2629,6 +2725,20 @@ static const char * const i2c3_groups[] = { "i2c3", }; +static const char * const intc_ex_groups[] = { + "intc_ex_irq0_a", + "intc_ex_irq0_b", + "intc_ex_irq1_a", + "intc_ex_irq1_b", + "intc_ex_irq2_a", + "intc_ex_irq2_b", + "intc_ex_irq3_a", + "intc_ex_irq3_b", + "intc_ex_irq4_a", + "intc_ex_irq4_b", + "intc_ex_irq5", +}; + static const char * const mmc_groups[] = { "mmc_data1", "mmc_data4", @@ -2813,6 +2923,8 @@ static const struct sh_pfc_function pinmux_functions[] = { SH_PFC_FUNCTION(i2c2), SH_PFC_FUNCTION(i2c3), + SH_PFC_FUNCTION(intc_ex), + SH_PFC_FUNCTION(mmc), SH_PFC_FUNCTION(msiof0), diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c index 20425afc6b33..c3256bfde502 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c +++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c @@ -892,6 +892,8 @@ static int rzg2l_set_power_source(struct rzg2l_pinctrl *pctrl, u32 pin, u32 caps val = PVDD_1800; break; case 2500: + if (!(caps & (PIN_CFG_IO_VMC_ETH0 | PIN_CFG_IO_VMC_ETH1))) + return -EINVAL; val = PVDD_2500; break; case 3300: @@ -2514,7 +2516,7 @@ static void rzg2l_pinctrl_pm_setup_dedicated_regs(struct rzg2l_pinctrl *pctrl, b } } -static void rzg2l_pinctrl_pm_setup_pfc(struct rzg2l_pinctrl *pctrl) +static void rzg2l_pinctrl_pm_setup_pfc(struct rzg2l_pinctrl *pctrl) { u32 nports = pctrl->data->n_port_pins / RZG2L_PINS_PER_PORT; const struct rzg2l_hwcfg *hwcfg = pctrl->data->hwcfg; diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.c b/drivers/pinctrl/samsung/pinctrl-exynos.c index 871c1eb46ddf..ce5e6783b5b9 100644 --- a/drivers/pinctrl/samsung/pinctrl-exynos.c +++ b/drivers/pinctrl/samsung/pinctrl-exynos.c @@ -13,6 +13,7 @@ // the Samsung pinctrl/gpiolib driver. It also includes the implementation of // external gpio and wakeup interrupt support. +#include <linux/clk.h> #include <linux/device.h> #include <linux/interrupt.h> #include <linux/irqdomain.h> @@ -61,6 +62,12 @@ static void exynos_irq_mask(struct irq_data *irqd) else reg_mask = our_chip->eint_mask + bank->eint_offset; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for masking IRQ\n"); + return; + } + raw_spin_lock_irqsave(&bank->slock, flags); mask = readl(bank->eint_base + reg_mask); @@ -68,6 +75,8 @@ static void exynos_irq_mask(struct irq_data *irqd) writel(mask, bank->eint_base + reg_mask); raw_spin_unlock_irqrestore(&bank->slock, flags); + + clk_disable(bank->drvdata->pclk); } static void exynos_irq_ack(struct irq_data *irqd) @@ -82,7 +91,15 @@ static void exynos_irq_ack(struct irq_data *irqd) else reg_pend = our_chip->eint_pend + bank->eint_offset; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock to ack IRQ\n"); + return; + } + writel(1 << irqd->hwirq, bank->eint_base + reg_pend); + + clk_disable(bank->drvdata->pclk); } static void exynos_irq_unmask(struct irq_data *irqd) @@ -110,6 +127,12 @@ static void exynos_irq_unmask(struct irq_data *irqd) else reg_mask = our_chip->eint_mask + bank->eint_offset; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for unmasking IRQ\n"); + return; + } + raw_spin_lock_irqsave(&bank->slock, flags); mask = readl(bank->eint_base + reg_mask); @@ -117,6 +140,8 @@ static void exynos_irq_unmask(struct irq_data *irqd) writel(mask, bank->eint_base + reg_mask); raw_spin_unlock_irqrestore(&bank->slock, flags); + + clk_disable(bank->drvdata->pclk); } static int exynos_irq_set_type(struct irq_data *irqd, unsigned int type) @@ -127,6 +152,7 @@ static int exynos_irq_set_type(struct irq_data *irqd, unsigned int type) unsigned int shift = EXYNOS_EINT_CON_LEN * irqd->hwirq; unsigned int con, trig_type; unsigned long reg_con; + int ret; switch (type) { case IRQ_TYPE_EDGE_RISING: @@ -159,11 +185,20 @@ static int exynos_irq_set_type(struct irq_data *irqd, unsigned int type) else reg_con = our_chip->eint_con + bank->eint_offset; + ret = clk_enable(bank->drvdata->pclk); + if (ret) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for configuring IRQ type\n"); + return ret; + } + con = readl(bank->eint_base + reg_con); con &= ~(EXYNOS_EINT_CON_MASK << shift); con |= trig_type << shift; writel(con, bank->eint_base + reg_con); + clk_disable(bank->drvdata->pclk); + return 0; } @@ -200,6 +235,14 @@ static int exynos_irq_request_resources(struct irq_data *irqd) shift = irqd->hwirq * bank_type->fld_width[PINCFG_TYPE_FUNC]; mask = (1 << bank_type->fld_width[PINCFG_TYPE_FUNC]) - 1; + ret = clk_enable(bank->drvdata->pclk); + if (ret) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for configuring pin %s-%lu\n", + bank->name, irqd->hwirq); + return ret; + } + raw_spin_lock_irqsave(&bank->slock, flags); con = readl(bank->pctl_base + reg_con); @@ -209,6 +252,8 @@ static int exynos_irq_request_resources(struct irq_data *irqd) raw_spin_unlock_irqrestore(&bank->slock, flags); + clk_disable(bank->drvdata->pclk); + return 0; } @@ -223,6 +268,13 @@ static void exynos_irq_release_resources(struct irq_data *irqd) shift = irqd->hwirq * bank_type->fld_width[PINCFG_TYPE_FUNC]; mask = (1 << bank_type->fld_width[PINCFG_TYPE_FUNC]) - 1; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for deconfiguring pin %s-%lu\n", + bank->name, irqd->hwirq); + return; + } + raw_spin_lock_irqsave(&bank->slock, flags); con = readl(bank->pctl_base + reg_con); @@ -232,6 +284,8 @@ static void exynos_irq_release_resources(struct irq_data *irqd) raw_spin_unlock_irqrestore(&bank->slock, flags); + clk_disable(bank->drvdata->pclk); + gpiochip_unlock_as_irq(&bank->gpio_chip, irqd->hwirq); } @@ -281,10 +335,19 @@ static irqreturn_t exynos_eint_gpio_irq(int irq, void *data) unsigned int svc, group, pin; int ret; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for handling IRQ\n"); + return IRQ_NONE; + } + if (bank->eint_con_offset) svc = readl(bank->eint_base + EXYNOSAUTO_SVC_OFFSET); else svc = readl(bank->eint_base + EXYNOS_SVC_OFFSET); + + clk_disable(bank->drvdata->pclk); + group = EXYNOS_SVC_GROUP(svc); pin = svc & EXYNOS_SVC_NUM_MASK; @@ -563,6 +626,20 @@ static void exynos_irq_demux_eint16_31(struct irq_desc *desc) chained_irq_enter(chip, desc); + /* + * just enable the clock once here, to avoid an enable/disable dance for + * each bank. + */ + if (eintd->nr_banks) { + struct samsung_pin_bank *b = eintd->banks[0]; + + if (clk_enable(b->drvdata->pclk)) { + dev_err(b->gpio_chip.parent, + "unable to enable clock for pending IRQs\n"); + return; + } + } + for (i = 0; i < eintd->nr_banks; ++i) { struct samsung_pin_bank *b = eintd->banks[i]; pend = readl(b->eint_base + b->irq_chip->eint_pend @@ -572,6 +649,9 @@ static void exynos_irq_demux_eint16_31(struct irq_desc *desc) exynos_irq_demux_eint(pend & ~mask, b->irq_domain); } + if (eintd->nr_banks) + clk_disable(eintd->banks[0]->drvdata->pclk); + chained_irq_exit(chip, desc); } @@ -695,6 +775,12 @@ static void exynos_pinctrl_suspend_bank( struct exynos_eint_gpio_save *save = bank->soc_priv; const void __iomem *regs = bank->eint_base; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for saving state\n"); + return; + } + save->eint_con = readl(regs + EXYNOS_GPIO_ECON_OFFSET + bank->eint_offset); save->eint_fltcon0 = readl(regs + EXYNOS_GPIO_EFLTCON_OFFSET @@ -704,6 +790,8 @@ static void exynos_pinctrl_suspend_bank( save->eint_mask = readl(regs + bank->irq_chip->eint_mask + bank->eint_offset); + clk_disable(bank->drvdata->pclk); + pr_debug("%s: save con %#010x\n", bank->name, save->eint_con); pr_debug("%s: save fltcon0 %#010x\n", bank->name, save->eint_fltcon0); pr_debug("%s: save fltcon1 %#010x\n", bank->name, save->eint_fltcon1); @@ -716,9 +804,17 @@ static void exynosauto_pinctrl_suspend_bank(struct samsung_pinctrl_drv_data *drv struct exynos_eint_gpio_save *save = bank->soc_priv; const void __iomem *regs = bank->eint_base; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for saving state\n"); + return; + } + save->eint_con = readl(regs + bank->pctl_offset + bank->eint_con_offset); save->eint_mask = readl(regs + bank->pctl_offset + bank->eint_mask_offset); + clk_disable(bank->drvdata->pclk); + pr_debug("%s: save con %#010x\n", bank->name, save->eint_con); pr_debug("%s: save mask %#010x\n", bank->name, save->eint_mask); } @@ -753,6 +849,12 @@ static void exynos_pinctrl_resume_bank( struct exynos_eint_gpio_save *save = bank->soc_priv; void __iomem *regs = bank->eint_base; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for restoring state\n"); + return; + } + pr_debug("%s: con %#010x => %#010x\n", bank->name, readl(regs + EXYNOS_GPIO_ECON_OFFSET + bank->eint_offset), save->eint_con); @@ -774,6 +876,8 @@ static void exynos_pinctrl_resume_bank( + 2 * bank->eint_offset + 4); writel(save->eint_mask, regs + bank->irq_chip->eint_mask + bank->eint_offset); + + clk_disable(bank->drvdata->pclk); } static void exynosauto_pinctrl_resume_bank(struct samsung_pinctrl_drv_data *drvdata, @@ -782,6 +886,12 @@ static void exynosauto_pinctrl_resume_bank(struct samsung_pinctrl_drv_data *drvd struct exynos_eint_gpio_save *save = bank->soc_priv; void __iomem *regs = bank->eint_base; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for restoring state\n"); + return; + } + pr_debug("%s: con %#010x => %#010x\n", bank->name, readl(regs + bank->pctl_offset + bank->eint_con_offset), save->eint_con); pr_debug("%s: mask %#010x => %#010x\n", bank->name, @@ -789,6 +899,8 @@ static void exynosauto_pinctrl_resume_bank(struct samsung_pinctrl_drv_data *drvd writel(save->eint_con, regs + bank->pctl_offset + bank->eint_con_offset); writel(save->eint_mask, regs + bank->pctl_offset + bank->eint_mask_offset); + + clk_disable(bank->drvdata->pclk); } void exynos_pinctrl_resume(struct samsung_pinctrl_drv_data *drvdata) diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c index ed07e23e0912..623df65a5d6f 100644 --- a/drivers/pinctrl/samsung/pinctrl-samsung.c +++ b/drivers/pinctrl/samsung/pinctrl-samsung.c @@ -15,6 +15,7 @@ // but provides extensions to which platform specific implementation of the gpio // and wakeup interrupts can be hooked to. +#include <linux/clk.h> #include <linux/err.h> #include <linux/gpio/driver.h> #include <linux/init.h> @@ -371,8 +372,8 @@ static void pin_to_reg_bank(struct samsung_pinctrl_drv_data *drvdata, } /* enable or disable a pinmux function */ -static void samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector, - unsigned group) +static int samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector, + unsigned group) { struct samsung_pinctrl_drv_data *drvdata; const struct samsung_pin_bank_type *type; @@ -382,6 +383,7 @@ static void samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector, unsigned long flags; const struct samsung_pmx_func *func; const struct samsung_pin_group *grp; + int ret; drvdata = pinctrl_dev_get_drvdata(pctldev); func = &drvdata->pmx_functions[selector]; @@ -397,6 +399,12 @@ static void samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector, reg += 4; } + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(pctldev->dev, "failed to enable clock for setup\n"); + return ret; + } + raw_spin_lock_irqsave(&bank->slock, flags); data = readl(reg + type->reg_offset[PINCFG_TYPE_FUNC]); @@ -405,6 +413,10 @@ static void samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector, writel(data, reg + type->reg_offset[PINCFG_TYPE_FUNC]); raw_spin_unlock_irqrestore(&bank->slock, flags); + + clk_disable(drvdata->pclk); + + return 0; } /* enable a specified pinmux by writing to registers */ @@ -412,8 +424,7 @@ static int samsung_pinmux_set_mux(struct pinctrl_dev *pctldev, unsigned selector, unsigned group) { - samsung_pinmux_setup(pctldev, selector, group); - return 0; + return samsung_pinmux_setup(pctldev, selector, group); } /* list of pinmux callbacks for the pinmux vertical in pinctrl core */ @@ -436,6 +447,7 @@ static int samsung_pinconf_rw(struct pinctrl_dev *pctldev, unsigned int pin, u32 data, width, pin_offset, mask, shift; u32 cfg_value, cfg_reg; unsigned long flags; + int ret; drvdata = pinctrl_dev_get_drvdata(pctldev); pin_to_reg_bank(drvdata, pin, ®_base, &pin_offset, &bank); @@ -447,6 +459,12 @@ static int samsung_pinconf_rw(struct pinctrl_dev *pctldev, unsigned int pin, width = type->fld_width[cfg_type]; cfg_reg = type->reg_offset[cfg_type]; + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(drvdata->dev, "failed to enable clock\n"); + return ret; + } + raw_spin_lock_irqsave(&bank->slock, flags); mask = (1 << width) - 1; @@ -466,6 +484,8 @@ static int samsung_pinconf_rw(struct pinctrl_dev *pctldev, unsigned int pin, raw_spin_unlock_irqrestore(&bank->slock, flags); + clk_disable(drvdata->pclk); + return 0; } @@ -555,11 +575,19 @@ static void samsung_gpio_set_value(struct gpio_chip *gc, static void samsung_gpio_set(struct gpio_chip *gc, unsigned offset, int value) { struct samsung_pin_bank *bank = gpiochip_get_data(gc); + struct samsung_pinctrl_drv_data *drvdata = bank->drvdata; unsigned long flags; + if (clk_enable(drvdata->pclk)) { + dev_err(drvdata->dev, "failed to enable clock\n"); + return; + } + raw_spin_lock_irqsave(&bank->slock, flags); samsung_gpio_set_value(gc, offset, value); raw_spin_unlock_irqrestore(&bank->slock, flags); + + clk_disable(drvdata->pclk); } /* gpiolib gpio_get callback function */ @@ -569,12 +597,23 @@ static int samsung_gpio_get(struct gpio_chip *gc, unsigned offset) u32 data; struct samsung_pin_bank *bank = gpiochip_get_data(gc); const struct samsung_pin_bank_type *type = bank->type; + struct samsung_pinctrl_drv_data *drvdata = bank->drvdata; + int ret; reg = bank->pctl_base + bank->pctl_offset; + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(drvdata->dev, "failed to enable clock\n"); + return ret; + } + data = readl(reg + type->reg_offset[PINCFG_TYPE_DAT]); data >>= offset; data &= 1; + + clk_disable(drvdata->pclk); + return data; } @@ -619,12 +658,22 @@ static int samsung_gpio_set_direction(struct gpio_chip *gc, static int samsung_gpio_direction_input(struct gpio_chip *gc, unsigned offset) { struct samsung_pin_bank *bank = gpiochip_get_data(gc); + struct samsung_pinctrl_drv_data *drvdata = bank->drvdata; unsigned long flags; int ret; + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(drvdata->dev, "failed to enable clock\n"); + return ret; + } + raw_spin_lock_irqsave(&bank->slock, flags); ret = samsung_gpio_set_direction(gc, offset, true); raw_spin_unlock_irqrestore(&bank->slock, flags); + + clk_disable(drvdata->pclk); + return ret; } @@ -633,14 +682,23 @@ static int samsung_gpio_direction_output(struct gpio_chip *gc, unsigned offset, int value) { struct samsung_pin_bank *bank = gpiochip_get_data(gc); + struct samsung_pinctrl_drv_data *drvdata = bank->drvdata; unsigned long flags; int ret; + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(drvdata->dev, "failed to enable clock\n"); + return ret; + } + raw_spin_lock_irqsave(&bank->slock, flags); samsung_gpio_set_value(gc, offset, value); ret = samsung_gpio_set_direction(gc, offset, false); raw_spin_unlock_irqrestore(&bank->slock, flags); + clk_disable(drvdata->pclk); + return ret; } @@ -1164,6 +1222,12 @@ static int samsung_pinctrl_probe(struct platform_device *pdev) } } + drvdata->pclk = devm_clk_get_optional_prepared(dev, "pclk"); + if (IS_ERR(drvdata->pclk)) { + ret = PTR_ERR(drvdata->pclk); + goto err_put_banks; + } + ret = samsung_pinctrl_register(pdev, drvdata); if (ret) goto err_put_banks; @@ -1202,6 +1266,13 @@ static int __maybe_unused samsung_pinctrl_suspend(struct device *dev) struct samsung_pinctrl_drv_data *drvdata = dev_get_drvdata(dev); int i; + i = clk_enable(drvdata->pclk); + if (i) { + dev_err(drvdata->dev, + "failed to enable clock for saving state\n"); + return i; + } + for (i = 0; i < drvdata->nr_banks; i++) { struct samsung_pin_bank *bank = &drvdata->pin_banks[i]; const void __iomem *reg = bank->pctl_base + bank->pctl_offset; @@ -1231,6 +1302,8 @@ static int __maybe_unused samsung_pinctrl_suspend(struct device *dev) } } + clk_disable(drvdata->pclk); + if (drvdata->suspend) drvdata->suspend(drvdata); if (drvdata->retention_ctrl && drvdata->retention_ctrl->enable) @@ -1250,8 +1323,20 @@ static int __maybe_unused samsung_pinctrl_suspend(struct device *dev) static int __maybe_unused samsung_pinctrl_resume(struct device *dev) { struct samsung_pinctrl_drv_data *drvdata = dev_get_drvdata(dev); + int ret; int i; + /* + * enable clock before the callback, as we don't want to have to deal + * with callback cleanup on clock failures. + */ + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(drvdata->dev, + "failed to enable clock for restoring state\n"); + return ret; + } + if (drvdata->resume) drvdata->resume(drvdata); @@ -1286,6 +1371,8 @@ static int __maybe_unused samsung_pinctrl_resume(struct device *dev) writel(bank->pm_save[type], reg + offs[type]); } + clk_disable(drvdata->pclk); + if (drvdata->retention_ctrl && drvdata->retention_ctrl->disable) drvdata->retention_ctrl->disable(drvdata); diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.h b/drivers/pinctrl/samsung/pinctrl-samsung.h index ab791afaabf5..d50ba6f07d5d 100644 --- a/drivers/pinctrl/samsung/pinctrl-samsung.h +++ b/drivers/pinctrl/samsung/pinctrl-samsung.h @@ -274,6 +274,7 @@ struct samsung_pin_ctrl { * through samsung_pinctrl_drv_data, not samsung_pin_bank). * @dev: device instance representing the controller. * @irq: interrpt number used by the controller to notify gpio interrupts. + * @pclk: optional bus clock if required for accessing registers * @ctrl: pin controller instance managed by the driver. * @pctl: pin controller descriptor registered with the pinctrl subsystem. * @pctl_dev: cookie representing pinctrl device instance. @@ -293,6 +294,7 @@ struct samsung_pinctrl_drv_data { void __iomem *virt_base; struct device *dev; int irq; + struct clk *pclk; struct pinctrl_desc pctl; struct pinctrl_dev *pctl_dev; diff --git a/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c b/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c index 919b6a20af83..5b4822f77d2a 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c +++ b/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c @@ -169,7 +169,6 @@ static struct platform_driver sun9i_a80_r_pinctrl_driver = { .probe = sun9i_a80_r_pinctrl_probe, .driver = { .name = "sun9i-a80-r-pinctrl", - .owner = THIS_MODULE, .of_match_table = sun9i_a80_r_pinctrl_match, }, }; diff --git a/drivers/platform/x86/intel_ips.c b/drivers/platform/x86/intel_ips.c index ba38649cc142..73ec4460a151 100644 --- a/drivers/platform/x86/intel_ips.c +++ b/drivers/platform/x86/intel_ips.c @@ -1505,7 +1505,7 @@ static int ips_probe(struct pci_dev *dev, const struct pci_device_id *id) * IRQ handler for ME interaction * Note: don't use MSI here as the PCH has bugs. */ - ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_LEGACY); + ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_INTX); if (ret < 0) return ret; diff --git a/drivers/platform/x86/uv_sysfs.c b/drivers/platform/x86/uv_sysfs.c index 3f6d52dea5c9..37372d7cc54a 100644 --- a/drivers/platform/x86/uv_sysfs.c +++ b/drivers/platform/x86/uv_sysfs.c @@ -11,6 +11,7 @@ #include <linux/device.h> #include <linux/slab.h> #include <linux/kobject.h> +#include <linux/vmalloc.h> #include <asm/uv/bios.h> #include <asm/uv/uv.h> #include <asm/uv/uv_hub.h> diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index abca56834468..750fda543308 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1545,7 +1545,8 @@ static int bq27xxx_battery_read_soc(struct bq27xxx_device_info *di) * Return a battery charge value in µAh * Or < 0 if something fails. */ -static int bq27xxx_battery_read_charge(struct bq27xxx_device_info *di, u8 reg) +static int bq27xxx_battery_read_charge(struct bq27xxx_device_info *di, u8 reg, + union power_supply_propval *val) { int charge; @@ -1561,34 +1562,39 @@ static int bq27xxx_battery_read_charge(struct bq27xxx_device_info *di, u8 reg) else charge *= 1000; - return charge; + val->intval = charge; + + return 0; } /* * Return the battery Nominal available capacity in µAh * Or < 0 if something fails. */ -static inline int bq27xxx_battery_read_nac(struct bq27xxx_device_info *di) +static inline int bq27xxx_battery_read_nac(struct bq27xxx_device_info *di, + union power_supply_propval *val) { - return bq27xxx_battery_read_charge(di, BQ27XXX_REG_NAC); + return bq27xxx_battery_read_charge(di, BQ27XXX_REG_NAC, val); } /* * Return the battery Remaining Capacity in µAh * Or < 0 if something fails. */ -static inline int bq27xxx_battery_read_rc(struct bq27xxx_device_info *di) +static inline int bq27xxx_battery_read_rc(struct bq27xxx_device_info *di, + union power_supply_propval *val) { - return bq27xxx_battery_read_charge(di, BQ27XXX_REG_RC); + return bq27xxx_battery_read_charge(di, BQ27XXX_REG_RC, val); } /* * Return the battery Full Charge Capacity in µAh * Or < 0 if something fails. */ -static inline int bq27xxx_battery_read_fcc(struct bq27xxx_device_info *di) +static inline int bq27xxx_battery_read_fcc(struct bq27xxx_device_info *di, + union power_supply_propval *val) { - return bq27xxx_battery_read_charge(di, BQ27XXX_REG_FCC); + return bq27xxx_battery_read_charge(di, BQ27XXX_REG_FCC, val); } /* @@ -1633,7 +1639,8 @@ static int bq27xxx_battery_read_dcap(struct bq27xxx_device_info *di, * Return the battery Available energy in µWh * Or < 0 if something fails. */ -static int bq27xxx_battery_read_energy(struct bq27xxx_device_info *di) +static int bq27xxx_battery_read_energy(struct bq27xxx_device_info *di, + union power_supply_propval *val) { int ae; @@ -1648,14 +1655,17 @@ static int bq27xxx_battery_read_energy(struct bq27xxx_device_info *di) else ae *= 1000; - return ae; + val->intval = ae; + + return 0; } /* - * Return the battery temperature in tenths of degree Kelvin + * Return the battery temperature in tenths of degree Celsius * Or < 0 if something fails. */ -static int bq27xxx_battery_read_temperature(struct bq27xxx_device_info *di) +static int bq27xxx_battery_read_temperature(struct bq27xxx_device_info *di, + union power_supply_propval *val) { int temp; @@ -1668,14 +1678,20 @@ static int bq27xxx_battery_read_temperature(struct bq27xxx_device_info *di) if (di->opts & BQ27XXX_O_ZERO) temp = 5 * temp / 2; - return temp; + /* Convert decidegree Kelvin to Celsius */ + temp -= 2731; + + val->intval = temp; + + return 0; } /* * Return the battery Cycle count total * Or < 0 if something fails. */ -static int bq27xxx_battery_read_cyct(struct bq27xxx_device_info *di) +static int bq27xxx_battery_read_cyct(struct bq27xxx_device_info *di, + union power_supply_propval *val) { int cyct; @@ -1683,14 +1699,17 @@ static int bq27xxx_battery_read_cyct(struct bq27xxx_device_info *di) if (cyct < 0) dev_err(di->dev, "error reading cycle count total\n"); - return cyct; + val->intval = cyct; + + return 0; } /* * Read a time register. * Return < 0 if something fails. */ -static int bq27xxx_battery_read_time(struct bq27xxx_device_info *di, u8 reg) +static int bq27xxx_battery_read_time(struct bq27xxx_device_info *di, u8 reg, + union power_supply_propval *val) { int tval; @@ -1704,7 +1723,9 @@ static int bq27xxx_battery_read_time(struct bq27xxx_device_info *di, u8 reg) if (tval == 65535) return -ENODATA; - return tval * 60; + val->intval = tval * 60; + + return 0; } /* @@ -1756,19 +1777,26 @@ static bool bq27xxx_battery_capacity_inaccurate(struct bq27xxx_device_info *di, return false; } -static int bq27xxx_battery_read_health(struct bq27xxx_device_info *di) +static int bq27xxx_battery_read_health(struct bq27xxx_device_info *di, + union power_supply_propval *val) { + int health; + /* Unlikely but important to return first */ if (unlikely(bq27xxx_battery_overtemp(di, di->cache.flags))) - return POWER_SUPPLY_HEALTH_OVERHEAT; - if (unlikely(bq27xxx_battery_undertemp(di, di->cache.flags))) - return POWER_SUPPLY_HEALTH_COLD; - if (unlikely(bq27xxx_battery_dead(di, di->cache.flags))) - return POWER_SUPPLY_HEALTH_DEAD; - if (unlikely(bq27xxx_battery_capacity_inaccurate(di, di->cache.flags))) - return POWER_SUPPLY_HEALTH_CALIBRATION_REQUIRED; - - return POWER_SUPPLY_HEALTH_GOOD; + health = POWER_SUPPLY_HEALTH_OVERHEAT; + else if (unlikely(bq27xxx_battery_undertemp(di, di->cache.flags))) + health = POWER_SUPPLY_HEALTH_COLD; + else if (unlikely(bq27xxx_battery_dead(di, di->cache.flags))) + health = POWER_SUPPLY_HEALTH_DEAD; + else if (unlikely(bq27xxx_battery_capacity_inaccurate(di, di->cache.flags))) + health = POWER_SUPPLY_HEALTH_CALIBRATION_REQUIRED; + else + health = POWER_SUPPLY_HEALTH_GOOD; + + val->intval = health; + + return 0; } static bool bq27xxx_battery_is_full(struct bq27xxx_device_info *di, int flags) @@ -1851,22 +1879,8 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) if ((cache.flags & 0xff) == 0xff) cache.flags = -1; /* read error */ if (cache.flags >= 0) { - cache.temperature = bq27xxx_battery_read_temperature(di); - if (di->regs[BQ27XXX_REG_TTE] != INVALID_REG_ADDR) - cache.time_to_empty = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTE); - if (di->regs[BQ27XXX_REG_TTECP] != INVALID_REG_ADDR) - cache.time_to_empty_avg = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTECP); - if (di->regs[BQ27XXX_REG_TTF] != INVALID_REG_ADDR) - cache.time_to_full = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTF); - - cache.charge_full = bq27xxx_battery_read_fcc(di); cache.capacity = bq27xxx_battery_read_soc(di); - if (di->regs[BQ27XXX_REG_AE] != INVALID_REG_ADDR) - cache.energy = bq27xxx_battery_read_energy(di); di->cache.flags = cache.flags; - cache.health = bq27xxx_battery_read_health(di); - if (di->regs[BQ27XXX_REG_CYCT] != INVALID_REG_ADDR) - cache.cycle_count = bq27xxx_battery_read_cyct(di); /* * On gauges with signed current reporting the current must be @@ -2038,18 +2052,16 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, ret = bq27xxx_battery_capacity_level(di, val); break; case POWER_SUPPLY_PROP_TEMP: - ret = bq27xxx_simple_value(di->cache.temperature, val); - if (ret == 0) - val->intval -= 2731; /* convert decidegree k to c */ + ret = bq27xxx_battery_read_temperature(di, val); break; case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: - ret = bq27xxx_simple_value(di->cache.time_to_empty, val); + ret = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTE, val); break; case POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG: - ret = bq27xxx_simple_value(di->cache.time_to_empty_avg, val); + ret = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTECP, val); break; case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW: - ret = bq27xxx_simple_value(di->cache.time_to_full, val); + ret = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTF, val); break; case POWER_SUPPLY_PROP_TECHNOLOGY: if (di->opts & BQ27XXX_O_MUL_CHEM) @@ -2059,12 +2071,12 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, break; case POWER_SUPPLY_PROP_CHARGE_NOW: if (di->regs[BQ27XXX_REG_NAC] != INVALID_REG_ADDR) - ret = bq27xxx_simple_value(bq27xxx_battery_read_nac(di), val); + ret = bq27xxx_battery_read_nac(di, val); else - ret = bq27xxx_simple_value(bq27xxx_battery_read_rc(di), val); + ret = bq27xxx_battery_read_rc(di, val); break; case POWER_SUPPLY_PROP_CHARGE_FULL: - ret = bq27xxx_simple_value(di->cache.charge_full, val); + ret = bq27xxx_battery_read_fcc(di, val); break; case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: ret = bq27xxx_battery_read_dcap(di, val); @@ -2077,16 +2089,16 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: return -EINVAL; case POWER_SUPPLY_PROP_CYCLE_COUNT: - ret = bq27xxx_simple_value(di->cache.cycle_count, val); + ret = bq27xxx_battery_read_cyct(di, val); break; case POWER_SUPPLY_PROP_ENERGY_NOW: - ret = bq27xxx_simple_value(di->cache.energy, val); + ret = bq27xxx_battery_read_energy(di, val); break; case POWER_SUPPLY_PROP_POWER_AVG: ret = bq27xxx_battery_pwr_avg(di, val); break; case POWER_SUPPLY_PROP_HEALTH: - ret = bq27xxx_simple_value(di->cache.health, val); + ret = bq27xxx_battery_read_health(di, val); break; case POWER_SUPPLY_PROP_MANUFACTURER: val->strval = BQ27XXX_MANUFACTURER; diff --git a/drivers/power/supply/cros_peripheral_charger.c b/drivers/power/supply/cros_peripheral_charger.c index a204f2355be4..d406f2a78449 100644 --- a/drivers/power/supply/cros_peripheral_charger.c +++ b/drivers/power/supply/cros_peripheral_charger.c @@ -5,6 +5,7 @@ * Copyright 2020 Google LLC. */ +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/notifier.h> #include <linux/platform_data/cros_ec_commands.h> @@ -367,16 +368,22 @@ static int __maybe_unused cros_pchg_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(cros_pchg_pm_ops, NULL, cros_pchg_resume); +static const struct platform_device_id cros_pchg_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_pchg_id); + static struct platform_driver cros_pchg_driver = { .driver = { .name = DRV_NAME, .pm = &cros_pchg_pm_ops, }, - .probe = cros_pchg_probe + .probe = cros_pchg_probe, + .id_table = cros_pchg_id, }; module_platform_driver(cros_pchg_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("ChromeOS EC peripheral device charger"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/power/supply/cros_usbpd-charger.c b/drivers/power/supply/cros_usbpd-charger.c index b6c96376776a..8008e31c0c09 100644 --- a/drivers/power/supply/cros_usbpd-charger.c +++ b/drivers/power/supply/cros_usbpd-charger.c @@ -5,6 +5,7 @@ * Copyright (c) 2014 - 2018 Google, Inc */ +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> @@ -711,16 +712,22 @@ static int cros_usbpd_charger_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(cros_usbpd_charger_pm_ops, NULL, cros_usbpd_charger_resume); +static const struct platform_device_id cros_usbpd_charger_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_usbpd_charger_id); + static struct platform_driver cros_usbpd_charger_driver = { .driver = { .name = DRV_NAME, .pm = &cros_usbpd_charger_pm_ops, }, - .probe = cros_usbpd_charger_probe + .probe = cros_usbpd_charger_probe, + .id_table = cros_usbpd_charger_id, }; module_platform_driver(cros_usbpd_charger_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("ChromeOS EC USBPD charger"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c index 0d2c3724d0bc..b86e11bdc07e 100644 --- a/drivers/power/supply/power_supply_sysfs.c +++ b/drivers/power/supply/power_supply_sysfs.c @@ -271,23 +271,6 @@ static ssize_t power_supply_show_usb_type(struct device *dev, return count; } -static ssize_t power_supply_show_charge_behaviour(struct device *dev, - struct power_supply *psy, - union power_supply_propval *value, - char *buf) -{ - int ret; - - ret = power_supply_get_property(psy, - POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, - value); - if (ret < 0) - return ret; - - return power_supply_charge_behaviour_show(dev, psy->desc->charge_behaviours, - value->intval, buf); -} - static ssize_t power_supply_show_property(struct device *dev, struct device_attribute *attr, char *buf) { @@ -321,7 +304,8 @@ static ssize_t power_supply_show_property(struct device *dev, &value, buf); break; case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: - ret = power_supply_show_charge_behaviour(dev, psy, &value, buf); + ret = power_supply_charge_behaviour_show(dev, psy->desc->charge_behaviours, + value.intval, buf); break; case POWER_SUPPLY_PROP_MODEL_NAME ... POWER_SUPPLY_PROP_SERIAL_NUMBER: ret = sysfs_emit(buf, "%s\n", value.strval); diff --git a/drivers/power/supply/sbs-manager.c b/drivers/power/supply/sbs-manager.c index 9e4141cffbf9..933b04806d10 100644 --- a/drivers/power/supply/sbs-manager.c +++ b/drivers/power/supply/sbs-manager.c @@ -358,7 +358,7 @@ static int sbsm_probe(struct i2c_client *client) /* register muxed i2c channels. One for each supported battery */ for (i = 0; i < SBSM_MAX_BATS; ++i) { if (data->supported_bats & BIT(i)) { - ret = i2c_mux_add_adapter(data->muxc, 0, i + 1, 0); + ret = i2c_mux_add_adapter(data->muxc, 0, i + 1); if (ret) break; } diff --git a/drivers/power/supply/test_power.c b/drivers/power/supply/test_power.c index 0d0a77584c5d..442ceb7795e1 100644 --- a/drivers/power/supply/test_power.c +++ b/drivers/power/supply/test_power.c @@ -35,6 +35,8 @@ static int battery_capacity = 50; static int battery_voltage = 3300; static int battery_charge_counter = -1000; static int battery_current = -1600; +static enum power_supply_charge_behaviour battery_charge_behaviour = + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO; static bool module_initialized; @@ -123,6 +125,9 @@ static int test_power_get_battery_property(struct power_supply *psy, case POWER_SUPPLY_PROP_CURRENT_NOW: val->intval = battery_current; break; + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + val->intval = battery_charge_behaviour; + break; default: pr_info("%s: some properties deliberately report errors.\n", __func__); @@ -131,6 +136,31 @@ static int test_power_get_battery_property(struct power_supply *psy, return 0; } +static int test_power_battery_property_is_writeable(struct power_supply *psy, + enum power_supply_property psp) +{ + return psp == POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR; +} + +static int test_power_set_battery_property(struct power_supply *psy, + enum power_supply_property psp, + const union power_supply_propval *val) +{ + switch (psp) { + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + if (val->intval < 0 || + val->intval >= BITS_PER_TYPE(typeof(psy->desc->charge_behaviours)) || + !(BIT(val->intval) & psy->desc->charge_behaviours)) { + return -EINVAL; + } + battery_charge_behaviour = val->intval; + break; + default: + return -EINVAL; + } + return 0; +} + static enum power_supply_property test_power_ac_props[] = { POWER_SUPPLY_PROP_ONLINE, }; @@ -156,6 +186,7 @@ static enum power_supply_property test_power_battery_props[] = { POWER_SUPPLY_PROP_VOLTAGE_NOW, POWER_SUPPLY_PROP_CURRENT_AVG, POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, }; static char *test_power_ac_supplied_to[] = { @@ -178,6 +209,11 @@ static const struct power_supply_desc test_power_desc[] = { .properties = test_power_battery_props, .num_properties = ARRAY_SIZE(test_power_battery_props), .get_property = test_power_get_battery_property, + .set_property = test_power_set_battery_property, + .property_is_writeable = test_power_battery_property_is_writeable, + .charge_behaviours = BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO) + | BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE) + | BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE), }, [TEST_USB] = { .name = "test_usb", diff --git a/drivers/pps/clients/pps_parport.c b/drivers/pps/clients/pps_parport.c index 42f93d4c6ee3..af972cdc04b5 100644 --- a/drivers/pps/clients/pps_parport.c +++ b/drivers/pps/clients/pps_parport.c @@ -148,7 +148,7 @@ static void parport_attach(struct parport *port) return; } - index = ida_simple_get(&pps_client_index, 0, 0, GFP_KERNEL); + index = ida_alloc(&pps_client_index, GFP_KERNEL); memset(&pps_client_cb, 0, sizeof(pps_client_cb)); pps_client_cb.private = device; pps_client_cb.irq_func = parport_irq; @@ -188,7 +188,7 @@ err_release_dev: err_unregister_dev: parport_unregister_device(device->pardev); err_free: - ida_simple_remove(&pps_client_index, index); + ida_free(&pps_client_index, index); kfree(device); } @@ -208,7 +208,7 @@ static void parport_detach(struct parport *port) pps_unregister_source(device->pps); parport_release(pardev); parport_unregister_device(pardev); - ida_simple_remove(&pps_client_index, device->index); + ida_free(&pps_client_index, device->index); kfree(device); } diff --git a/drivers/rapidio/Kconfig b/drivers/rapidio/Kconfig index b9f8514909bf..f1d742ac8f7c 100644 --- a/drivers/rapidio/Kconfig +++ b/drivers/rapidio/Kconfig @@ -59,26 +59,13 @@ config RAPIDIO_DEBUG If you are unsure about this, say N here. -choice - prompt "Enumeration method" - depends on RAPIDIO - default RAPIDIO_ENUM_BASIC - help - There are different enumeration and discovery mechanisms offered - for RapidIO subsystem. You may select single built-in method or - or any number of methods to be built as modules. - Selecting a built-in method disables use of loadable methods. - - If unsure, select Basic built-in. - config RAPIDIO_ENUM_BASIC - tristate "Basic" + tristate "Basic Enumeration method" + depends on RAPIDIO help This option includes basic RapidIO fabric enumeration and discovery mechanism similar to one described in RapidIO specification Annex 1. -endchoice - config RAPIDIO_CHMAN tristate "RapidIO Channelized Messaging driver" depends on RAPIDIO diff --git a/drivers/remoteproc/mtk_common.h b/drivers/remoteproc/mtk_common.h index 6d7736a031f7..fd5c539ab2ac 100644 --- a/drivers/remoteproc/mtk_common.h +++ b/drivers/remoteproc/mtk_common.h @@ -78,7 +78,6 @@ #define MT8195_L2TCM_OFFSET 0x850d0 #define SCP_FW_VER_LEN 32 -#define SCP_SHARE_BUFFER_SIZE 288 struct scp_run { u32 signaled; @@ -97,6 +96,11 @@ struct scp_ipi_desc { struct mtk_scp; +struct mtk_scp_sizes_data { + size_t max_dram_size; + size_t ipi_share_buffer_size; +}; + struct mtk_scp_of_data { int (*scp_clk_get)(struct mtk_scp *scp); int (*scp_before_load)(struct mtk_scp *scp); @@ -110,6 +114,7 @@ struct mtk_scp_of_data { u32 host_to_scp_int_bit; size_t ipi_buf_offset; + const struct mtk_scp_sizes_data *scp_sizes; }; struct mtk_scp_of_cluster { @@ -141,10 +146,10 @@ struct mtk_scp { struct scp_ipi_desc ipi_desc[SCP_IPI_MAX]; bool ipi_id_ack[SCP_IPI_MAX]; wait_queue_head_t ack_wq; + u8 *share_buf; void *cpu_addr; dma_addr_t dma_addr; - size_t dram_size; struct rproc_subdev *rpmsg_subdev; @@ -162,7 +167,7 @@ struct mtk_scp { struct mtk_share_obj { u32 id; u32 len; - u8 share_buf[SCP_SHARE_BUFFER_SIZE]; + u8 *share_buf; }; void scp_memcpy_aligned(void __iomem *dst, const void *src, unsigned int len); diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index a35409eda0cf..b8498772dba1 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -20,7 +20,6 @@ #include "mtk_common.h" #include "remoteproc_internal.h" -#define MAX_CODE_SIZE 0x500000 #define SECTION_NAME_IPI_BUFFER ".ipi_buffer" /** @@ -94,14 +93,15 @@ static void scp_ipi_handler(struct mtk_scp *scp) { struct mtk_share_obj __iomem *rcv_obj = scp->recv_buf; struct scp_ipi_desc *ipi_desc = scp->ipi_desc; - u8 tmp_data[SCP_SHARE_BUFFER_SIZE]; scp_ipi_handler_t handler; u32 id = readl(&rcv_obj->id); u32 len = readl(&rcv_obj->len); + const struct mtk_scp_sizes_data *scp_sizes; - if (len > SCP_SHARE_BUFFER_SIZE) { - dev_err(scp->dev, "ipi message too long (len %d, max %d)", len, - SCP_SHARE_BUFFER_SIZE); + scp_sizes = scp->data->scp_sizes; + if (len > scp_sizes->ipi_share_buffer_size) { + dev_err(scp->dev, "ipi message too long (len %d, max %zd)", len, + scp_sizes->ipi_share_buffer_size); return; } if (id >= SCP_IPI_MAX) { @@ -117,8 +117,9 @@ static void scp_ipi_handler(struct mtk_scp *scp) return; } - memcpy_fromio(tmp_data, &rcv_obj->share_buf, len); - handler(tmp_data, len, ipi_desc[id].priv); + memset(scp->share_buf, 0, scp_sizes->ipi_share_buffer_size); + memcpy_fromio(scp->share_buf, &rcv_obj->share_buf, len); + handler(scp->share_buf, len, ipi_desc[id].priv); scp_ipi_unlock(scp, id); scp->ipi_id_ack[id] = true; @@ -132,7 +133,9 @@ static int scp_elf_read_ipi_buf_addr(struct mtk_scp *scp, static int scp_ipi_init(struct mtk_scp *scp, const struct firmware *fw) { int ret; - size_t offset; + size_t buf_sz, offset; + size_t share_buf_offset; + const struct mtk_scp_sizes_data *scp_sizes; /* read the ipi buf addr from FW itself first */ ret = scp_elf_read_ipi_buf_addr(scp, fw, &offset); @@ -144,12 +147,23 @@ static int scp_ipi_init(struct mtk_scp *scp, const struct firmware *fw) } dev_info(scp->dev, "IPI buf addr %#010zx\n", offset); + /* Make sure IPI buffer fits in the L2TCM range assigned to this core */ + buf_sz = sizeof(*scp->recv_buf) + sizeof(*scp->send_buf); + + if (scp->sram_size < buf_sz + offset) { + dev_err(scp->dev, "IPI buffer does not fit in SRAM.\n"); + return -EOVERFLOW; + } + + scp_sizes = scp->data->scp_sizes; scp->recv_buf = (struct mtk_share_obj __iomem *) (scp->sram_base + offset); + share_buf_offset = sizeof(scp->recv_buf->id) + + sizeof(scp->recv_buf->len) + scp_sizes->ipi_share_buffer_size; scp->send_buf = (struct mtk_share_obj __iomem *) - (scp->sram_base + offset + sizeof(*scp->recv_buf)); - memset_io(scp->recv_buf, 0, sizeof(*scp->recv_buf)); - memset_io(scp->send_buf, 0, sizeof(*scp->send_buf)); + (scp->sram_base + offset + share_buf_offset); + memset_io(scp->recv_buf, 0, share_buf_offset); + memset_io(scp->send_buf, 0, share_buf_offset); return 0; } @@ -463,6 +477,86 @@ static int mt8186_scp_before_load(struct mtk_scp *scp) return 0; } +static int mt8188_scp_l2tcm_on(struct mtk_scp *scp) +{ + struct mtk_scp_of_cluster *scp_cluster = scp->cluster; + + mutex_lock(&scp_cluster->cluster_lock); + + if (scp_cluster->l2tcm_refcnt == 0) { + /* clear SPM interrupt, SCP2SPM_IPC_CLR */ + writel(0xff, scp->cluster->reg_base + MT8192_SCP2SPM_IPC_CLR); + + /* Power on L2TCM */ + scp_sram_power_on(scp->cluster->reg_base + MT8192_L2TCM_SRAM_PD_0, 0); + scp_sram_power_on(scp->cluster->reg_base + MT8192_L2TCM_SRAM_PD_1, 0); + scp_sram_power_on(scp->cluster->reg_base + MT8192_L2TCM_SRAM_PD_2, 0); + scp_sram_power_on(scp->cluster->reg_base + MT8192_L1TCM_SRAM_PDN, 0); + } + + scp_cluster->l2tcm_refcnt += 1; + + mutex_unlock(&scp_cluster->cluster_lock); + + return 0; +} + +static int mt8188_scp_before_load(struct mtk_scp *scp) +{ + writel(1, scp->cluster->reg_base + MT8192_CORE0_SW_RSTN_SET); + + mt8188_scp_l2tcm_on(scp); + + scp_sram_power_on(scp->cluster->reg_base + MT8192_CPU0_SRAM_PD, 0); + + /* enable MPU for all memory regions */ + writel(0xff, scp->cluster->reg_base + MT8192_CORE0_MEM_ATT_PREDEF); + + return 0; +} + +static int mt8188_scp_c1_before_load(struct mtk_scp *scp) +{ + u32 sec_ctrl; + struct mtk_scp *scp_c0; + struct mtk_scp_of_cluster *scp_cluster = scp->cluster; + + scp->data->scp_reset_assert(scp); + + mt8188_scp_l2tcm_on(scp); + + scp_sram_power_on(scp->cluster->reg_base + MT8195_CPU1_SRAM_PD, 0); + + /* enable MPU for all memory regions */ + writel(0xff, scp->cluster->reg_base + MT8195_CORE1_MEM_ATT_PREDEF); + + /* + * The L2TCM_OFFSET_RANGE and L2TCM_OFFSET shift the destination address + * on SRAM when SCP core 1 accesses SRAM. + * + * This configuration solves booting the SCP core 0 and core 1 from + * different SRAM address because core 0 and core 1 both boot from + * the head of SRAM by default. this must be configured before boot SCP core 1. + * + * The value of L2TCM_OFFSET_RANGE is from the viewpoint of SCP core 1. + * When SCP core 1 issues address within the range (L2TCM_OFFSET_RANGE), + * the address will be added with a fixed offset (L2TCM_OFFSET) on the bus. + * The shift action is tranparent to software. + */ + writel(0, scp->cluster->reg_base + MT8195_L2TCM_OFFSET_RANGE_0_LOW); + writel(scp->sram_size, scp->cluster->reg_base + MT8195_L2TCM_OFFSET_RANGE_0_HIGH); + + scp_c0 = list_first_entry(&scp_cluster->mtk_scp_list, struct mtk_scp, elem); + writel(scp->sram_phys - scp_c0->sram_phys, scp->cluster->reg_base + MT8195_L2TCM_OFFSET); + + /* enable SRAM offset when fetching instruction and data */ + sec_ctrl = readl(scp->cluster->reg_base + MT8195_SEC_CTRL); + sec_ctrl |= MT8195_CORE_OFFSET_ENABLE_I | MT8195_CORE_OFFSET_ENABLE_D; + writel(sec_ctrl, scp->cluster->reg_base + MT8195_SEC_CTRL); + + return 0; +} + static int mt8192_scp_before_load(struct mtk_scp *scp) { /* clear SPM interrupt, SCP2SPM_IPC_CLR */ @@ -653,14 +747,16 @@ stop: static void *mt8183_scp_da_to_va(struct mtk_scp *scp, u64 da, size_t len) { int offset; + const struct mtk_scp_sizes_data *scp_sizes; + scp_sizes = scp->data->scp_sizes; if (da < scp->sram_size) { offset = da; if (offset >= 0 && (offset + len) <= scp->sram_size) return (void __force *)scp->sram_base + offset; - } else if (scp->dram_size) { + } else if (scp_sizes->max_dram_size) { offset = da - scp->dma_addr; - if (offset >= 0 && (offset + len) <= scp->dram_size) + if (offset >= 0 && (offset + len) <= scp_sizes->max_dram_size) return scp->cpu_addr + offset; } @@ -670,7 +766,9 @@ static void *mt8183_scp_da_to_va(struct mtk_scp *scp, u64 da, size_t len) static void *mt8192_scp_da_to_va(struct mtk_scp *scp, u64 da, size_t len) { int offset; + const struct mtk_scp_sizes_data *scp_sizes; + scp_sizes = scp->data->scp_sizes; if (da >= scp->sram_phys && (da + len) <= scp->sram_phys + scp->sram_size) { offset = da - scp->sram_phys; @@ -686,9 +784,9 @@ static void *mt8192_scp_da_to_va(struct mtk_scp *scp, u64 da, size_t len) } /* optional memory region */ - if (scp->dram_size && + if (scp_sizes->max_dram_size && da >= scp->dma_addr && - (da + len) <= scp->dma_addr + scp->dram_size) { + (da + len) <= scp->dma_addr + scp_sizes->max_dram_size) { offset = da - scp->dma_addr; return scp->cpu_addr + offset; } @@ -709,6 +807,47 @@ static void mt8183_scp_stop(struct mtk_scp *scp) writel(0, scp->cluster->reg_base + MT8183_WDT_CFG); } +static void mt8188_scp_l2tcm_off(struct mtk_scp *scp) +{ + struct mtk_scp_of_cluster *scp_cluster = scp->cluster; + + mutex_lock(&scp_cluster->cluster_lock); + + if (scp_cluster->l2tcm_refcnt > 0) + scp_cluster->l2tcm_refcnt -= 1; + + if (scp_cluster->l2tcm_refcnt == 0) { + /* Power off L2TCM */ + scp_sram_power_off(scp->cluster->reg_base + MT8192_L2TCM_SRAM_PD_0, 0); + scp_sram_power_off(scp->cluster->reg_base + MT8192_L2TCM_SRAM_PD_1, 0); + scp_sram_power_off(scp->cluster->reg_base + MT8192_L2TCM_SRAM_PD_2, 0); + scp_sram_power_off(scp->cluster->reg_base + MT8192_L1TCM_SRAM_PDN, 0); + } + + mutex_unlock(&scp_cluster->cluster_lock); +} + +static void mt8188_scp_stop(struct mtk_scp *scp) +{ + mt8188_scp_l2tcm_off(scp); + + scp_sram_power_off(scp->cluster->reg_base + MT8192_CPU0_SRAM_PD, 0); + + /* Disable SCP watchdog */ + writel(0, scp->cluster->reg_base + MT8192_CORE0_WDT_CFG); +} + +static void mt8188_scp_c1_stop(struct mtk_scp *scp) +{ + mt8188_scp_l2tcm_off(scp); + + /* Power off CPU SRAM */ + scp_sram_power_off(scp->cluster->reg_base + MT8195_CPU1_SRAM_PD, 0); + + /* Disable SCP watchdog */ + writel(0, scp->cluster->reg_base + MT8195_CORE1_WDT_CFG); +} + static void mt8192_scp_stop(struct mtk_scp *scp) { /* Disable SRAM clock */ @@ -868,6 +1007,7 @@ EXPORT_SYMBOL_GPL(scp_mapping_dm_addr); static int scp_map_memory_region(struct mtk_scp *scp) { int ret; + const struct mtk_scp_sizes_data *scp_sizes; ret = of_reserved_mem_device_init(scp->dev); @@ -883,8 +1023,8 @@ static int scp_map_memory_region(struct mtk_scp *scp) } /* Reserved SCP code size */ - scp->dram_size = MAX_CODE_SIZE; - scp->cpu_addr = dma_alloc_coherent(scp->dev, scp->dram_size, + scp_sizes = scp->data->scp_sizes; + scp->cpu_addr = dma_alloc_coherent(scp->dev, scp_sizes->max_dram_size, &scp->dma_addr, GFP_KERNEL); if (!scp->cpu_addr) return -ENOMEM; @@ -894,10 +1034,13 @@ static int scp_map_memory_region(struct mtk_scp *scp) static void scp_unmap_memory_region(struct mtk_scp *scp) { - if (scp->dram_size == 0) + const struct mtk_scp_sizes_data *scp_sizes; + + scp_sizes = scp->data->scp_sizes; + if (scp_sizes->max_dram_size == 0) return; - dma_free_coherent(scp->dev, scp->dram_size, scp->cpu_addr, + dma_free_coherent(scp->dev, scp_sizes->max_dram_size, scp->cpu_addr, scp->dma_addr); of_reserved_mem_device_release(scp->dev); } @@ -961,6 +1104,7 @@ static struct mtk_scp *scp_rproc_init(struct platform_device *pdev, struct resource *res; const char *fw_name = "scp.img"; int ret, i; + const struct mtk_scp_sizes_data *scp_sizes; ret = rproc_of_parse_firmware(dev, 0, &fw_name); if (ret < 0 && ret != -EINVAL) @@ -1008,6 +1152,14 @@ static struct mtk_scp *scp_rproc_init(struct platform_device *pdev, goto release_dev_mem; } + scp_sizes = scp->data->scp_sizes; + scp->share_buf = kzalloc(scp_sizes->ipi_share_buffer_size, GFP_KERNEL); + if (!scp->share_buf) { + dev_err(dev, "Failed to allocate IPI share buffer\n"); + ret = -ENOMEM; + goto release_dev_mem; + } + init_waitqueue_head(&scp->run.wq); init_waitqueue_head(&scp->ack_wq); @@ -1027,6 +1179,8 @@ static struct mtk_scp *scp_rproc_init(struct platform_device *pdev, remove_subdev: scp_remove_rpmsg_subdev(scp); scp_ipi_unregister(scp, SCP_IPI_INIT); + kfree(scp->share_buf); + scp->share_buf = NULL; release_dev_mem: scp_unmap_memory_region(scp); for (i = 0; i < SCP_IPI_MAX; i++) @@ -1042,6 +1196,8 @@ static void scp_free(struct mtk_scp *scp) scp_remove_rpmsg_subdev(scp); scp_ipi_unregister(scp, SCP_IPI_INIT); + kfree(scp->share_buf); + scp->share_buf = NULL; scp_unmap_memory_region(scp); for (i = 0; i < SCP_IPI_MAX; i++) mutex_destroy(&scp->ipi_desc[i].lock); @@ -1228,6 +1384,21 @@ static void scp_remove(struct platform_device *pdev) mutex_destroy(&scp_cluster->cluster_lock); } +static const struct mtk_scp_sizes_data default_scp_sizes = { + .max_dram_size = 0x500000, + .ipi_share_buffer_size = 288, +}; + +static const struct mtk_scp_sizes_data mt8188_scp_sizes = { + .max_dram_size = 0x500000, + .ipi_share_buffer_size = 600, +}; + +static const struct mtk_scp_sizes_data mt8188_scp_c1_sizes = { + .max_dram_size = 0xA00000, + .ipi_share_buffer_size = 600, +}; + static const struct mtk_scp_of_data mt8183_of_data = { .scp_clk_get = mt8183_scp_clk_get, .scp_before_load = mt8183_scp_before_load, @@ -1239,6 +1410,7 @@ static const struct mtk_scp_of_data mt8183_of_data = { .host_to_scp_reg = MT8183_HOST_TO_SCP, .host_to_scp_int_bit = MT8183_HOST_IPC_INT_BIT, .ipi_buf_offset = 0x7bdb0, + .scp_sizes = &default_scp_sizes, }; static const struct mtk_scp_of_data mt8186_of_data = { @@ -1252,18 +1424,33 @@ static const struct mtk_scp_of_data mt8186_of_data = { .host_to_scp_reg = MT8183_HOST_TO_SCP, .host_to_scp_int_bit = MT8183_HOST_IPC_INT_BIT, .ipi_buf_offset = 0x3bdb0, + .scp_sizes = &default_scp_sizes, }; static const struct mtk_scp_of_data mt8188_of_data = { .scp_clk_get = mt8195_scp_clk_get, - .scp_before_load = mt8192_scp_before_load, - .scp_irq_handler = mt8192_scp_irq_handler, + .scp_before_load = mt8188_scp_before_load, + .scp_irq_handler = mt8195_scp_irq_handler, .scp_reset_assert = mt8192_scp_reset_assert, .scp_reset_deassert = mt8192_scp_reset_deassert, - .scp_stop = mt8192_scp_stop, + .scp_stop = mt8188_scp_stop, .scp_da_to_va = mt8192_scp_da_to_va, .host_to_scp_reg = MT8192_GIPC_IN_SET, .host_to_scp_int_bit = MT8192_HOST_IPC_INT_BIT, + .scp_sizes = &mt8188_scp_sizes, +}; + +static const struct mtk_scp_of_data mt8188_of_data_c1 = { + .scp_clk_get = mt8195_scp_clk_get, + .scp_before_load = mt8188_scp_c1_before_load, + .scp_irq_handler = mt8195_scp_c1_irq_handler, + .scp_reset_assert = mt8195_scp_c1_reset_assert, + .scp_reset_deassert = mt8195_scp_c1_reset_deassert, + .scp_stop = mt8188_scp_c1_stop, + .scp_da_to_va = mt8192_scp_da_to_va, + .host_to_scp_reg = MT8192_GIPC_IN_SET, + .host_to_scp_int_bit = MT8195_CORE1_HOST_IPC_INT_BIT, + .scp_sizes = &mt8188_scp_c1_sizes, }; static const struct mtk_scp_of_data mt8192_of_data = { @@ -1276,6 +1463,7 @@ static const struct mtk_scp_of_data mt8192_of_data = { .scp_da_to_va = mt8192_scp_da_to_va, .host_to_scp_reg = MT8192_GIPC_IN_SET, .host_to_scp_int_bit = MT8192_HOST_IPC_INT_BIT, + .scp_sizes = &default_scp_sizes, }; static const struct mtk_scp_of_data mt8195_of_data = { @@ -1288,6 +1476,7 @@ static const struct mtk_scp_of_data mt8195_of_data = { .scp_da_to_va = mt8192_scp_da_to_va, .host_to_scp_reg = MT8192_GIPC_IN_SET, .host_to_scp_int_bit = MT8192_HOST_IPC_INT_BIT, + .scp_sizes = &default_scp_sizes, }; static const struct mtk_scp_of_data mt8195_of_data_c1 = { @@ -1300,6 +1489,13 @@ static const struct mtk_scp_of_data mt8195_of_data_c1 = { .scp_da_to_va = mt8192_scp_da_to_va, .host_to_scp_reg = MT8192_GIPC_IN_SET, .host_to_scp_int_bit = MT8195_CORE1_HOST_IPC_INT_BIT, + .scp_sizes = &default_scp_sizes, +}; + +static const struct mtk_scp_of_data *mt8188_of_data_cores[] = { + &mt8188_of_data, + &mt8188_of_data_c1, + NULL }; static const struct mtk_scp_of_data *mt8195_of_data_cores[] = { @@ -1312,6 +1508,7 @@ static const struct of_device_id mtk_scp_of_match[] = { { .compatible = "mediatek,mt8183-scp", .data = &mt8183_of_data }, { .compatible = "mediatek,mt8186-scp", .data = &mt8186_of_data }, { .compatible = "mediatek,mt8188-scp", .data = &mt8188_of_data }, + { .compatible = "mediatek,mt8188-scp-dual", .data = &mt8188_of_data_cores }, { .compatible = "mediatek,mt8192-scp", .data = &mt8192_of_data }, { .compatible = "mediatek,mt8195-scp", .data = &mt8195_of_data }, { .compatible = "mediatek,mt8195-scp-dual", .data = &mt8195_of_data_cores }, diff --git a/drivers/remoteproc/mtk_scp_ipi.c b/drivers/remoteproc/mtk_scp_ipi.c index cd0b60106ec2..c068227e251e 100644 --- a/drivers/remoteproc/mtk_scp_ipi.c +++ b/drivers/remoteproc/mtk_scp_ipi.c @@ -162,10 +162,13 @@ int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, struct mtk_share_obj __iomem *send_obj = scp->send_buf; u32 val; int ret; + const struct mtk_scp_sizes_data *scp_sizes; + + scp_sizes = scp->data->scp_sizes; if (WARN_ON(id <= SCP_IPI_INIT) || WARN_ON(id >= SCP_IPI_MAX) || WARN_ON(id == SCP_IPI_NS_SERVICE) || - WARN_ON(len > sizeof(send_obj->share_buf)) || WARN_ON(!buf)) + WARN_ON(len > scp_sizes->ipi_share_buffer_size) || WARN_ON(!buf)) return -EINVAL; ret = clk_prepare_enable(scp->clk); @@ -184,7 +187,7 @@ int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, goto unlock_mutex; } - scp_memcpy_aligned(send_obj->share_buf, buf, len); + scp_memcpy_aligned(&send_obj->share_buf, buf, len); writel(len, &send_obj->len); writel(id, &send_obj->id); diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h index f62a82d71dfa..0cd09e67ac14 100644 --- a/drivers/remoteproc/remoteproc_internal.h +++ b/drivers/remoteproc/remoteproc_internal.h @@ -72,7 +72,7 @@ void rproc_init_debugfs(void); void rproc_exit_debugfs(void); /* from remoteproc_sysfs.c */ -extern struct class rproc_class; +extern const struct class rproc_class; int rproc_init_sysfs(void); void rproc_exit_sysfs(void); diff --git a/drivers/remoteproc/remoteproc_sysfs.c b/drivers/remoteproc/remoteproc_sysfs.c index 8c7ea8922638..138e752c5e4e 100644 --- a/drivers/remoteproc/remoteproc_sysfs.c +++ b/drivers/remoteproc/remoteproc_sysfs.c @@ -254,7 +254,7 @@ static const struct attribute_group *rproc_devgroups[] = { NULL }; -struct class rproc_class = { +const struct class rproc_class = { .name = "remoteproc", .dev_groups = rproc_devgroups, }; diff --git a/drivers/remoteproc/ti_k3_r5_remoteproc.c b/drivers/remoteproc/ti_k3_r5_remoteproc.c index ad3415a3851b..50e486bcfa10 100644 --- a/drivers/remoteproc/ti_k3_r5_remoteproc.c +++ b/drivers/remoteproc/ti_k3_r5_remoteproc.c @@ -103,12 +103,14 @@ struct k3_r5_soc_data { * @dev: cached device pointer * @mode: Mode to configure the Cluster - Split or LockStep * @cores: list of R5 cores within the cluster + * @core_transition: wait queue to sync core state changes * @soc_data: SoC-specific feature data for a R5FSS */ struct k3_r5_cluster { struct device *dev; enum cluster_mode mode; struct list_head cores; + wait_queue_head_t core_transition; const struct k3_r5_soc_data *soc_data; }; @@ -128,6 +130,7 @@ struct k3_r5_cluster { * @atcm_enable: flag to control ATCM enablement * @btcm_enable: flag to control BTCM enablement * @loczrama: flag to dictate which TCM is at device address 0x0 + * @released_from_reset: flag to signal when core is out of reset */ struct k3_r5_core { struct list_head elem; @@ -144,6 +147,7 @@ struct k3_r5_core { u32 atcm_enable; u32 btcm_enable; u32 loczrama; + bool released_from_reset; }; /** @@ -460,6 +464,8 @@ static int k3_r5_rproc_prepare(struct rproc *rproc) ret); return ret; } + core->released_from_reset = true; + wake_up_interruptible(&cluster->core_transition); /* * Newer IP revisions like on J7200 SoCs support h/w auto-initialization @@ -542,7 +548,7 @@ static int k3_r5_rproc_start(struct rproc *rproc) struct k3_r5_rproc *kproc = rproc->priv; struct k3_r5_cluster *cluster = kproc->cluster; struct device *dev = kproc->dev; - struct k3_r5_core *core; + struct k3_r5_core *core0, *core; u32 boot_addr; int ret; @@ -568,6 +574,16 @@ static int k3_r5_rproc_start(struct rproc *rproc) goto unroll_core_run; } } else { + /* do not allow core 1 to start before core 0 */ + core0 = list_first_entry(&cluster->cores, struct k3_r5_core, + elem); + if (core != core0 && core0->rproc->state == RPROC_OFFLINE) { + dev_err(dev, "%s: can not start core 1 before core 0\n", + __func__); + ret = -EPERM; + goto put_mbox; + } + ret = k3_r5_core_run(core); if (ret) goto put_mbox; @@ -613,7 +629,8 @@ static int k3_r5_rproc_stop(struct rproc *rproc) { struct k3_r5_rproc *kproc = rproc->priv; struct k3_r5_cluster *cluster = kproc->cluster; - struct k3_r5_core *core = kproc->core; + struct device *dev = kproc->dev; + struct k3_r5_core *core1, *core = kproc->core; int ret; /* halt all applicable cores */ @@ -626,6 +643,16 @@ static int k3_r5_rproc_stop(struct rproc *rproc) } } } else { + /* do not allow core 0 to stop before core 1 */ + core1 = list_last_entry(&cluster->cores, struct k3_r5_core, + elem); + if (core != core1 && core1->rproc->state != RPROC_OFFLINE) { + dev_err(dev, "%s: can not stop core 0 before core 1\n", + __func__); + ret = -EPERM; + goto out; + } + ret = k3_r5_core_halt(core); if (ret) goto out; @@ -1140,6 +1167,12 @@ static int k3_r5_rproc_configure_mode(struct k3_r5_rproc *kproc) return ret; } + /* + * Skip the waiting mechanism for sequential power-on of cores if the + * core has already been booted by another entity. + */ + core->released_from_reset = c_state; + ret = ti_sci_proc_get_status(core->tsp, &boot_vec, &cfg, &ctrl, &stat); if (ret < 0) { @@ -1280,6 +1313,26 @@ init_rmem: cluster->mode == CLUSTER_MODE_SINGLECPU || cluster->mode == CLUSTER_MODE_SINGLECORE) break; + + /* + * R5 cores require to be powered on sequentially, core0 + * should be in higher power state than core1 in a cluster + * So, wait for current core to power up before proceeding + * to next core and put timeout of 2sec for each core. + * + * This waiting mechanism is necessary because + * rproc_auto_boot_callback() for core1 can be called before + * core0 due to thread execution order. + */ + ret = wait_event_interruptible_timeout(cluster->core_transition, + core->released_from_reset, + msecs_to_jiffies(2000)); + if (ret <= 0) { + dev_err(dev, + "Timed out waiting for %s core to power up!\n", + rproc->name); + return ret; + } } return 0; @@ -1709,6 +1762,7 @@ static int k3_r5_probe(struct platform_device *pdev) cluster->dev = dev; cluster->soc_data = data; INIT_LIST_HEAD(&cluster->cores); + init_waitqueue_head(&cluster->core_transition); ret = of_property_read_u32(np, "ti,cluster-mode", &cluster->mode); if (ret < 0 && ret != -EINVAL) { diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c index 4395edea9a64..84243d1dff9f 100644 --- a/drivers/remoteproc/xlnx_r5_remoteproc.c +++ b/drivers/remoteproc/xlnx_r5_remoteproc.c @@ -74,8 +74,8 @@ struct mbox_info { }; /* - * Hardcoded TCM bank values. This will be removed once TCM bindings are - * accepted for system-dt specifications and upstreamed in linux kernel + * Hardcoded TCM bank values. This will stay in driver to maintain backward + * compatibility with device-tree that does not have TCM information. */ static const struct mem_bank_data zynqmp_tcm_banks_split[] = { {0xffe00000UL, 0x0, 0x10000UL, PD_R5_0_ATCM, "atcm0"}, /* TCM 64KB each */ @@ -84,12 +84,12 @@ static const struct mem_bank_data zynqmp_tcm_banks_split[] = { {0xffeb0000UL, 0x20000, 0x10000UL, PD_R5_1_BTCM, "btcm1"}, }; -/* In lockstep mode cluster combines each 64KB TCM and makes 128KB TCM */ +/* In lockstep mode cluster uses each 64KB TCM from second core as well */ static const struct mem_bank_data zynqmp_tcm_banks_lockstep[] = { - {0xffe00000UL, 0x0, 0x20000UL, PD_R5_0_ATCM, "atcm0"}, /* TCM 128KB each */ - {0xffe20000UL, 0x20000, 0x20000UL, PD_R5_0_BTCM, "btcm0"}, - {0, 0, 0, PD_R5_1_ATCM, ""}, - {0, 0, 0, PD_R5_1_BTCM, ""}, + {0xffe00000UL, 0x0, 0x10000UL, PD_R5_0_ATCM, "atcm0"}, /* TCM 64KB each */ + {0xffe20000UL, 0x20000, 0x10000UL, PD_R5_0_BTCM, "btcm0"}, + {0xffe10000UL, 0x10000, 0x10000UL, PD_R5_1_ATCM, "atcm1"}, + {0xffe30000UL, 0x30000, 0x10000UL, PD_R5_1_BTCM, "btcm1"}, }; /** @@ -301,36 +301,6 @@ static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid) } /* - * zynqmp_r5_set_mode() - * - * set RPU cluster and TCM operation mode - * - * @r5_core: pointer to zynqmp_r5_core type object - * @fw_reg_val: value expected by firmware to configure RPU cluster mode - * @tcm_mode: value expected by fw to configure TCM mode (lockstep or split) - * - * Return: 0 for success and < 0 for failure - */ -static int zynqmp_r5_set_mode(struct zynqmp_r5_core *r5_core, - enum rpu_oper_mode fw_reg_val, - enum rpu_tcm_comb tcm_mode) -{ - int ret; - - ret = zynqmp_pm_set_rpu_mode(r5_core->pm_domain_id, fw_reg_val); - if (ret < 0) { - dev_err(r5_core->dev, "failed to set RPU mode\n"); - return ret; - } - - ret = zynqmp_pm_set_tcm_config(r5_core->pm_domain_id, tcm_mode); - if (ret < 0) - dev_err(r5_core->dev, "failed to configure TCM\n"); - - return ret; -} - -/* * zynqmp_r5_rproc_start() * @rproc: single R5 core's corresponding rproc instance * @@ -486,6 +456,7 @@ static int add_mem_regions_carveout(struct rproc *rproc) } rproc_add_carveout(rproc, rproc_mem); + rproc_coredump_add_segment(rproc, rmem->base, rmem->size); dev_dbg(&rproc->dev, "reserved mem carveout %s addr=%llx, size=0x%llx", it.node->name, rmem->base, rmem->size); @@ -540,14 +511,14 @@ static int tcm_mem_map(struct rproc *rproc, } /* - * add_tcm_carveout_split_mode() + * add_tcm_banks() * @rproc: single R5 core's corresponding rproc instance * - * allocate and add remoteproc carveout for TCM memory in split mode + * allocate and add remoteproc carveout for TCM memory * * return 0 on success, otherwise non-zero value on failure */ -static int add_tcm_carveout_split_mode(struct rproc *rproc) +static int add_tcm_banks(struct rproc *rproc) { struct rproc_mem_entry *rproc_mem; struct zynqmp_r5_core *r5_core; @@ -580,10 +551,10 @@ static int add_tcm_carveout_split_mode(struct rproc *rproc) ZYNQMP_PM_REQUEST_ACK_BLOCKING); if (ret < 0) { dev_err(dev, "failed to turn on TCM 0x%x", pm_domain_id); - goto release_tcm_split; + goto release_tcm; } - dev_dbg(dev, "TCM carveout split mode %s addr=%llx, da=0x%x, size=0x%lx", + dev_dbg(dev, "TCM carveout %s addr=%llx, da=0x%x, size=0x%lx", bank_name, bank_addr, da, bank_size); rproc_mem = rproc_mem_entry_init(dev, NULL, bank_addr, @@ -593,15 +564,16 @@ static int add_tcm_carveout_split_mode(struct rproc *rproc) if (!rproc_mem) { ret = -ENOMEM; zynqmp_pm_release_node(pm_domain_id); - goto release_tcm_split; + goto release_tcm; } rproc_add_carveout(rproc, rproc_mem); + rproc_coredump_add_segment(rproc, da, bank_size); } return 0; -release_tcm_split: +release_tcm: /* If failed, Turn off all TCM banks turned on before */ for (i--; i >= 0; i--) { pm_domain_id = r5_core->tcm_banks[i]->pm_domain_id; @@ -611,127 +583,6 @@ release_tcm_split: } /* - * add_tcm_carveout_lockstep_mode() - * @rproc: single R5 core's corresponding rproc instance - * - * allocate and add remoteproc carveout for TCM memory in lockstep mode - * - * return 0 on success, otherwise non-zero value on failure - */ -static int add_tcm_carveout_lockstep_mode(struct rproc *rproc) -{ - struct rproc_mem_entry *rproc_mem; - struct zynqmp_r5_core *r5_core; - int i, num_banks, ret; - phys_addr_t bank_addr; - size_t bank_size = 0; - struct device *dev; - u32 pm_domain_id; - char *bank_name; - u32 da; - - r5_core = rproc->priv; - dev = r5_core->dev; - - /* Go through zynqmp banks for r5 node */ - num_banks = r5_core->tcm_bank_count; - - /* - * In lockstep mode, TCM is contiguous memory block - * However, each TCM block still needs to be enabled individually. - * So, Enable each TCM block individually. - * Although ATCM and BTCM is contiguous memory block, add two separate - * carveouts for both. - */ - for (i = 0; i < num_banks; i++) { - pm_domain_id = r5_core->tcm_banks[i]->pm_domain_id; - - /* Turn on each TCM bank individually */ - ret = zynqmp_pm_request_node(pm_domain_id, - ZYNQMP_PM_CAPABILITY_ACCESS, 0, - ZYNQMP_PM_REQUEST_ACK_BLOCKING); - if (ret < 0) { - dev_err(dev, "failed to turn on TCM 0x%x", pm_domain_id); - goto release_tcm_lockstep; - } - - bank_size = r5_core->tcm_banks[i]->size; - if (bank_size == 0) - continue; - - bank_addr = r5_core->tcm_banks[i]->addr; - da = r5_core->tcm_banks[i]->da; - bank_name = r5_core->tcm_banks[i]->bank_name; - - /* Register TCM address range, TCM map and unmap functions */ - rproc_mem = rproc_mem_entry_init(dev, NULL, bank_addr, - bank_size, da, - tcm_mem_map, tcm_mem_unmap, - bank_name); - if (!rproc_mem) { - ret = -ENOMEM; - zynqmp_pm_release_node(pm_domain_id); - goto release_tcm_lockstep; - } - - /* If registration is success, add carveouts */ - rproc_add_carveout(rproc, rproc_mem); - - dev_dbg(dev, "TCM carveout lockstep mode %s addr=0x%llx, da=0x%x, size=0x%lx", - bank_name, bank_addr, da, bank_size); - } - - return 0; - -release_tcm_lockstep: - /* If failed, Turn off all TCM banks turned on before */ - for (i--; i >= 0; i--) { - pm_domain_id = r5_core->tcm_banks[i]->pm_domain_id; - zynqmp_pm_release_node(pm_domain_id); - } - return ret; -} - -/* - * add_tcm_banks() - * @rproc: single R5 core's corresponding rproc instance - * - * allocate and add remoteproc carveouts for TCM memory based on cluster mode - * - * return 0 on success, otherwise non-zero value on failure - */ -static int add_tcm_banks(struct rproc *rproc) -{ - struct zynqmp_r5_cluster *cluster; - struct zynqmp_r5_core *r5_core; - struct device *dev; - - r5_core = rproc->priv; - if (!r5_core) - return -EINVAL; - - dev = r5_core->dev; - - cluster = dev_get_drvdata(dev->parent); - if (!cluster) { - dev_err(dev->parent, "Invalid driver data\n"); - return -EINVAL; - } - - /* - * In lockstep mode TCM banks are one contiguous memory region of 256Kb - * In split mode, each TCM bank is 64Kb and not contiguous. - * We add memory carveouts accordingly. - */ - if (cluster->mode == SPLIT_MODE) - return add_tcm_carveout_split_mode(rproc); - else if (cluster->mode == LOCKSTEP_MODE) - return add_tcm_carveout_lockstep_mode(rproc); - - return -EINVAL; -} - -/* * zynqmp_r5_parse_fw() * @rproc: single R5 core's corresponding rproc instance * @fw: ptr to firmware to be loaded onto r5 core @@ -853,6 +704,8 @@ static struct zynqmp_r5_core *zynqmp_r5_add_rproc_core(struct device *cdev) return ERR_PTR(-ENOMEM); } + rproc_coredump_set_elf_info(r5_rproc, ELFCLASS32, EM_ARM); + r5_rproc->auto_boot = false; r5_core = r5_rproc->priv; r5_core->dev = cdev; @@ -878,6 +731,103 @@ free_rproc: return ERR_PTR(ret); } +static int zynqmp_r5_get_tcm_node_from_dt(struct zynqmp_r5_cluster *cluster) +{ + int i, j, tcm_bank_count, ret, tcm_pd_idx, pd_count; + struct of_phandle_args out_args; + struct zynqmp_r5_core *r5_core; + struct platform_device *cpdev; + struct mem_bank_data *tcm; + struct device_node *np; + struct resource *res; + u64 abs_addr, size; + struct device *dev; + + for (i = 0; i < cluster->core_count; i++) { + r5_core = cluster->r5_cores[i]; + dev = r5_core->dev; + np = r5_core->np; + + pd_count = of_count_phandle_with_args(np, "power-domains", + "#power-domain-cells"); + + if (pd_count <= 0) { + dev_err(dev, "invalid power-domains property, %d\n", pd_count); + return -EINVAL; + } + + /* First entry in power-domains list is for r5 core, rest for TCM. */ + tcm_bank_count = pd_count - 1; + + if (tcm_bank_count <= 0) { + dev_err(dev, "invalid TCM count %d\n", tcm_bank_count); + return -EINVAL; + } + + r5_core->tcm_banks = devm_kcalloc(dev, tcm_bank_count, + sizeof(struct mem_bank_data *), + GFP_KERNEL); + if (!r5_core->tcm_banks) + return -ENOMEM; + + r5_core->tcm_bank_count = tcm_bank_count; + for (j = 0, tcm_pd_idx = 1; j < tcm_bank_count; j++, tcm_pd_idx++) { + tcm = devm_kzalloc(dev, sizeof(struct mem_bank_data), + GFP_KERNEL); + if (!tcm) + return -ENOMEM; + + r5_core->tcm_banks[j] = tcm; + + /* Get power-domains id of TCM. */ + ret = of_parse_phandle_with_args(np, "power-domains", + "#power-domain-cells", + tcm_pd_idx, &out_args); + if (ret) { + dev_err(r5_core->dev, + "failed to get tcm %d pm domain, ret %d\n", + tcm_pd_idx, ret); + return ret; + } + tcm->pm_domain_id = out_args.args[0]; + of_node_put(out_args.np); + + /* Get TCM address without translation. */ + ret = of_property_read_reg(np, j, &abs_addr, &size); + if (ret) { + dev_err(dev, "failed to get reg property\n"); + return ret; + } + + /* + * Remote processor can address only 32 bits + * so convert 64-bits into 32-bits. This will discard + * any unwanted upper 32-bits. + */ + tcm->da = (u32)abs_addr; + tcm->size = (u32)size; + + cpdev = to_platform_device(dev); + res = platform_get_resource(cpdev, IORESOURCE_MEM, j); + if (!res) { + dev_err(dev, "failed to get tcm resource\n"); + return -EINVAL; + } + + tcm->addr = (u32)res->start; + tcm->bank_name = (char *)res->name; + res = devm_request_mem_region(dev, tcm->addr, tcm->size, + tcm->bank_name); + if (!res) { + dev_err(dev, "failed to request tcm resource\n"); + return -EINVAL; + } + } + } + + return 0; +} + /** * zynqmp_r5_get_tcm_node() * Ideally this function should parse tcm node and store information @@ -954,11 +904,18 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster, { struct device *dev = cluster->dev; struct zynqmp_r5_core *r5_core; - int ret, i; + int ret = -EINVAL, i; - ret = zynqmp_r5_get_tcm_node(cluster); - if (ret < 0) { - dev_err(dev, "can't get tcm node, err %d\n", ret); + r5_core = cluster->r5_cores[0]; + + /* Maintain backward compatibility for zynqmp by using hardcode TCM address. */ + if (of_find_property(r5_core->np, "reg", NULL)) + ret = zynqmp_r5_get_tcm_node_from_dt(cluster); + else if (device_is_compatible(dev, "xlnx,zynqmp-r5fss")) + ret = zynqmp_r5_get_tcm_node(cluster); + + if (ret) { + dev_err(dev, "can't get tcm, err %d\n", ret); return ret; } @@ -973,12 +930,21 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster, return ret; } - ret = zynqmp_r5_set_mode(r5_core, fw_reg_val, tcm_mode); - if (ret) { - dev_err(dev, "failed to set r5 cluster mode %d, err %d\n", - cluster->mode, ret); + ret = zynqmp_pm_set_rpu_mode(r5_core->pm_domain_id, fw_reg_val); + if (ret < 0) { + dev_err(r5_core->dev, "failed to set RPU mode\n"); return ret; } + + if (of_find_property(dev_of_node(dev), "xlnx,tcm-mode", NULL) || + device_is_compatible(dev, "xlnx,zynqmp-r5fss")) { + ret = zynqmp_pm_set_tcm_config(r5_core->pm_domain_id, + tcm_mode); + if (ret < 0) { + dev_err(r5_core->dev, "failed to configure TCM\n"); + return ret; + } + } } return 0; @@ -1023,16 +989,27 @@ static int zynqmp_r5_cluster_init(struct zynqmp_r5_cluster *cluster) * fail driver probe if either of that is not set in dts. */ if (cluster_mode == LOCKSTEP_MODE) { - tcm_mode = PM_RPU_TCM_COMB; fw_reg_val = PM_RPU_MODE_LOCKSTEP; } else if (cluster_mode == SPLIT_MODE) { - tcm_mode = PM_RPU_TCM_SPLIT; fw_reg_val = PM_RPU_MODE_SPLIT; } else { dev_err(dev, "driver does not support cluster mode %d\n", cluster_mode); return -EINVAL; } + if (of_find_property(dev_node, "xlnx,tcm-mode", NULL)) { + ret = of_property_read_u32(dev_node, "xlnx,tcm-mode", (u32 *)&tcm_mode); + if (ret) + return ret; + } else if (device_is_compatible(dev, "xlnx,zynqmp-r5fss")) { + if (cluster_mode == LOCKSTEP_MODE) + tcm_mode = PM_RPU_TCM_COMB; + else + tcm_mode = PM_RPU_TCM_SPLIT; + } else { + tcm_mode = PM_RPU_TCM_COMB; + } + /* * Number of cores is decided by number of child nodes of * r5f subsystem node in dts. If Split mode is used in dts @@ -1216,6 +1193,8 @@ static int zynqmp_r5_remoteproc_probe(struct platform_device *pdev) /* Match table for OF platform binding */ static const struct of_device_id zynqmp_r5_remoteproc_match[] = { + { .compatible = "xlnx,versal-net-r52fss", }, + { .compatible = "xlnx,versal-r5fss", }, { .compatible = "xlnx,zynqmp-r5fss", }, { /* end of list */ }, }; diff --git a/drivers/reset/reset-mpfs.c b/drivers/reset/reset-mpfs.c index 7f3fb2d472f4..710f9c1676f9 100644 --- a/drivers/reset/reset-mpfs.c +++ b/drivers/reset/reset-mpfs.c @@ -8,9 +8,11 @@ */ #include <linux/auxiliary_bus.h> #include <linux/delay.h> +#include <linux/io.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> +#include <linux/slab.h> #include <linux/reset-controller.h> #include <dt-bindings/clock/microchip,mpfs-clock.h> #include <soc/microchip/mpfs.h> @@ -28,20 +30,30 @@ /* block concurrent access to the soft reset register */ static DEFINE_SPINLOCK(mpfs_reset_lock); +struct mpfs_reset { + void __iomem *base; + struct reset_controller_dev rcdev; +}; + +static inline struct mpfs_reset *to_mpfs_reset(struct reset_controller_dev *rcdev) +{ + return container_of(rcdev, struct mpfs_reset, rcdev); +} + /* * Peripheral clock resets */ - static int mpfs_assert(struct reset_controller_dev *rcdev, unsigned long id) { + struct mpfs_reset *rst = to_mpfs_reset(rcdev); unsigned long flags; u32 reg; spin_lock_irqsave(&mpfs_reset_lock, flags); - reg = mpfs_reset_read(rcdev->dev); + reg = readl(rst->base); reg |= BIT(id); - mpfs_reset_write(rcdev->dev, reg); + writel(reg, rst->base); spin_unlock_irqrestore(&mpfs_reset_lock, flags); @@ -50,14 +62,15 @@ static int mpfs_assert(struct reset_controller_dev *rcdev, unsigned long id) static int mpfs_deassert(struct reset_controller_dev *rcdev, unsigned long id) { + struct mpfs_reset *rst = to_mpfs_reset(rcdev); unsigned long flags; u32 reg; spin_lock_irqsave(&mpfs_reset_lock, flags); - reg = mpfs_reset_read(rcdev->dev); + reg = readl(rst->base); reg &= ~BIT(id); - mpfs_reset_write(rcdev->dev, reg); + writel(reg, rst->base); spin_unlock_irqrestore(&mpfs_reset_lock, flags); @@ -66,7 +79,8 @@ static int mpfs_deassert(struct reset_controller_dev *rcdev, unsigned long id) static int mpfs_status(struct reset_controller_dev *rcdev, unsigned long id) { - u32 reg = mpfs_reset_read(rcdev->dev); + struct mpfs_reset *rst = to_mpfs_reset(rcdev); + u32 reg = readl(rst->base); /* * It is safe to return here as MPFS_NUM_RESETS makes sure the sign bit @@ -121,11 +135,15 @@ static int mpfs_reset_probe(struct auxiliary_device *adev, { struct device *dev = &adev->dev; struct reset_controller_dev *rcdev; + struct mpfs_reset *rst; - rcdev = devm_kzalloc(dev, sizeof(*rcdev), GFP_KERNEL); - if (!rcdev) + rst = devm_kzalloc(dev, sizeof(*rst), GFP_KERNEL); + if (!rst) return -ENOMEM; + rst->base = (void __iomem *)adev->dev.platform_data; + + rcdev = &rst->rcdev; rcdev->dev = dev; rcdev->dev->parent = dev->parent; rcdev->ops = &mpfs_reset_ops; @@ -137,9 +155,68 @@ static int mpfs_reset_probe(struct auxiliary_device *adev, return devm_reset_controller_register(dev, rcdev); } +static void mpfs_reset_unregister_adev(void *_adev) +{ + struct auxiliary_device *adev = _adev; + + auxiliary_device_delete(adev); + auxiliary_device_uninit(adev); +} + +static void mpfs_reset_adev_release(struct device *dev) +{ + struct auxiliary_device *adev = to_auxiliary_dev(dev); + + kfree(adev); +} + +static struct auxiliary_device *mpfs_reset_adev_alloc(struct device *clk_dev) +{ + struct auxiliary_device *adev; + int ret; + + adev = kzalloc(sizeof(*adev), GFP_KERNEL); + if (!adev) + return ERR_PTR(-ENOMEM); + + adev->name = "reset-mpfs"; + adev->dev.parent = clk_dev; + adev->dev.release = mpfs_reset_adev_release; + adev->id = 666u; + + ret = auxiliary_device_init(adev); + if (ret) { + kfree(adev); + return ERR_PTR(ret); + } + + return adev; +} + +int mpfs_reset_controller_register(struct device *clk_dev, void __iomem *base) +{ + struct auxiliary_device *adev; + int ret; + + adev = mpfs_reset_adev_alloc(clk_dev); + if (IS_ERR(adev)) + return PTR_ERR(adev); + + ret = auxiliary_device_add(adev); + if (ret) { + auxiliary_device_uninit(adev); + return ret; + } + + adev->dev.platform_data = (__force void *)base; + + return devm_add_action_or_reset(clk_dev, mpfs_reset_unregister_adev, adev); +} +EXPORT_SYMBOL_NS_GPL(mpfs_reset_controller_register, MCHP_CLK_MPFS); + static const struct auxiliary_device_id mpfs_reset_ids[] = { { - .name = "clk_mpfs.reset-mpfs", + .name = "reset_mpfs.reset-mpfs", }, { } }; diff --git a/drivers/rpmsg/qcom_glink_ssr.c b/drivers/rpmsg/qcom_glink_ssr.c index 39ffa384c9b1..e71d3716c55c 100644 --- a/drivers/rpmsg/qcom_glink_ssr.c +++ b/drivers/rpmsg/qcom_glink_ssr.c @@ -154,6 +154,7 @@ static const struct rpmsg_device_id qcom_glink_ssr_match[] = { { "glink_ssr" }, {} }; +MODULE_DEVICE_TABLE(rpmsg, qcom_glink_ssr_match); static struct rpmsg_driver qcom_glink_ssr_driver = { .probe = qcom_glink_ssr_probe, diff --git a/drivers/rpmsg/rpmsg_char.c b/drivers/rpmsg/rpmsg_char.c index 1cb8d7474428..d7a342510902 100644 --- a/drivers/rpmsg/rpmsg_char.c +++ b/drivers/rpmsg/rpmsg_char.c @@ -423,7 +423,7 @@ static struct rpmsg_eptdev *rpmsg_chrdev_eptdev_alloc(struct rpmsg_device *rpdev init_waitqueue_head(&eptdev->readq); device_initialize(dev); - dev->class = rpmsg_class; + dev->class = &rpmsg_class; dev->parent = parent; dev->groups = rpmsg_eptdev_groups; dev_set_drvdata(dev, eptdev); diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c index 4295c01a2861..0fa08266404d 100644 --- a/drivers/rpmsg/rpmsg_core.c +++ b/drivers/rpmsg/rpmsg_core.c @@ -20,7 +20,9 @@ #include "rpmsg_internal.h" -struct class *rpmsg_class; +const struct class rpmsg_class = { + .name = "rpmsg", +}; EXPORT_SYMBOL(rpmsg_class); /** @@ -715,16 +717,16 @@ static int __init rpmsg_init(void) { int ret; - rpmsg_class = class_create("rpmsg"); - if (IS_ERR(rpmsg_class)) { - pr_err("failed to create rpmsg class\n"); - return PTR_ERR(rpmsg_class); + ret = class_register(&rpmsg_class); + if (ret) { + pr_err("failed to register rpmsg class\n"); + return ret; } ret = bus_register(&rpmsg_bus); if (ret) { pr_err("failed to register rpmsg bus: %d\n", ret); - class_destroy(rpmsg_class); + class_destroy(&rpmsg_class); } return ret; } @@ -733,7 +735,7 @@ postcore_initcall(rpmsg_init); static void __exit rpmsg_fini(void) { bus_unregister(&rpmsg_bus); - class_destroy(rpmsg_class); + class_destroy(&rpmsg_class); } module_exit(rpmsg_fini); diff --git a/drivers/rpmsg/rpmsg_ctrl.c b/drivers/rpmsg/rpmsg_ctrl.c index c312794ba4b3..28f57945ccd9 100644 --- a/drivers/rpmsg/rpmsg_ctrl.c +++ b/drivers/rpmsg/rpmsg_ctrl.c @@ -150,7 +150,7 @@ static int rpmsg_ctrldev_probe(struct rpmsg_device *rpdev) dev = &ctrldev->dev; device_initialize(dev); dev->parent = &rpdev->dev; - dev->class = rpmsg_class; + dev->class = &rpmsg_class; mutex_init(&ctrldev->ctrl_lock); cdev_init(&ctrldev->cdev, &rpmsg_ctrldev_fops); diff --git a/drivers/rpmsg/rpmsg_internal.h b/drivers/rpmsg/rpmsg_internal.h index b950d6f790a3..a3ba768138f1 100644 --- a/drivers/rpmsg/rpmsg_internal.h +++ b/drivers/rpmsg/rpmsg_internal.h @@ -18,7 +18,7 @@ #define to_rpmsg_device(d) container_of(d, struct rpmsg_device, dev) #define to_rpmsg_driver(d) container_of(d, struct rpmsg_driver, drv) -extern struct class *rpmsg_class; +extern const struct class rpmsg_class; /** * struct rpmsg_device_ops - indirection table for the rpmsg_device operations diff --git a/drivers/s390/block/dasd_ioctl.c b/drivers/s390/block/dasd_ioctl.c index 7e0ed7032f76..eb5dcbe37230 100644 --- a/drivers/s390/block/dasd_ioctl.c +++ b/drivers/s390/block/dasd_ioctl.c @@ -215,7 +215,7 @@ dasd_format(struct dasd_block *block, struct format_data_t *fdata) * enabling the device later. */ if (fdata->start_unit == 0) { - block->gdp->part0->bd_inode->i_blkbits = + block->gdp->part0->bd_mapping->host->i_blkbits = blksize_bits(fdata->blksize); } diff --git a/drivers/scsi/aic7xxx/Makefile b/drivers/scsi/aic7xxx/Makefile index e0188ecd85b2..853c72a81ae0 100644 --- a/drivers/scsi/aic7xxx/Makefile +++ b/drivers/scsi/aic7xxx/Makefile @@ -55,9 +55,9 @@ aicasm-7xxx-opts-$(CONFIG_AIC7XXX_REG_PRETTY_PRINT) := \ ifeq ($(CONFIG_AIC7XXX_BUILD_FIRMWARE),y) $(obj)/aic7xxx_seq.h: $(src)/aic7xxx.seq $(src)/aic7xxx.reg $(obj)/aicasm/aicasm - $(obj)/aicasm/aicasm -I$(srctree)/$(src) -r $(obj)/aic7xxx_reg.h \ + $(obj)/aicasm/aicasm -I $(src) -r $(obj)/aic7xxx_reg.h \ $(aicasm-7xxx-opts-y) -o $(obj)/aic7xxx_seq.h \ - $(srctree)/$(src)/aic7xxx.seq + $(src)/aic7xxx.seq $(aic7xxx-gen-y): $(objtree)/$(obj)/aic7xxx_seq.h @true @@ -73,9 +73,9 @@ aicasm-79xx-opts-$(CONFIG_AIC79XX_REG_PRETTY_PRINT) := \ ifeq ($(CONFIG_AIC79XX_BUILD_FIRMWARE),y) $(obj)/aic79xx_seq.h: $(src)/aic79xx.seq $(src)/aic79xx.reg $(obj)/aicasm/aicasm - $(obj)/aicasm/aicasm -I$(srctree)/$(src) -r $(obj)/aic79xx_reg.h \ + $(obj)/aicasm/aicasm -I $(src) -r $(obj)/aic79xx_reg.h \ $(aicasm-79xx-opts-y) -o $(obj)/aic79xx_seq.h \ - $(srctree)/$(src)/aic79xx.seq + $(src)/aic79xx.seq $(aic79xx-gen-y): $(objtree)/$(obj)/aic79xx_seq.h @true @@ -83,5 +83,5 @@ else $(obj)/aic79xx_reg_print.c: $(src)/aic79xx_reg_print.c_shipped endif -$(obj)/aicasm/aicasm: $(srctree)/$(src)/aicasm/*.[chyl] - $(MAKE) -C $(srctree)/$(src)/aicasm OUTDIR=$(shell pwd)/$(obj)/aicasm/ +$(obj)/aicasm/aicasm: $(src)/aicasm/*.[chyl] + $(MAKE) -C $(src)/aicasm OUTDIR=$(shell pwd)/$(obj)/aicasm/ diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c index ad227e6cb10e..35860c61468b 100644 --- a/drivers/scsi/arcmsr/arcmsr_hba.c +++ b/drivers/scsi/arcmsr/arcmsr_hba.c @@ -1007,7 +1007,7 @@ msi_int0: goto msi_int1; } } - nvec = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_LEGACY); + nvec = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_INTX); if (nvec < 1) return FAILED; msi_int1: diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c index 49c57a9c110b..e044ed09d7e0 100644 --- a/drivers/scsi/hpsa.c +++ b/drivers/scsi/hpsa.c @@ -7509,7 +7509,7 @@ fallback: */ static int hpsa_interrupt_mode(struct ctlr_info *h) { - unsigned int flags = PCI_IRQ_LEGACY; + unsigned int flags = PCI_IRQ_INTX; int ret; /* Some boards advertise MSI but don't really support it */ diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c index 388c8a10295a..31cf2d31cceb 100644 --- a/drivers/scsi/ipr.c +++ b/drivers/scsi/ipr.c @@ -9465,7 +9465,7 @@ static int ipr_probe_ioa(struct pci_dev *pdev, ipr_number_of_msix = IPR_MAX_MSIX_VECTORS; } - irq_flag = PCI_IRQ_LEGACY; + irq_flag = PCI_IRQ_INTX; if (ioa_cfg->ipr_chip->has_msi) irq_flag |= PCI_IRQ_MSI | PCI_IRQ_MSIX; rc = pci_alloc_irq_vectors(pdev, 1, ipr_number_of_msix, irq_flag); diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index def0d905b6d9..88acefbf9aea 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -6305,7 +6305,7 @@ static int megasas_init_fw(struct megasas_instance *instance) } if (!instance->msix_vectors) { - i = pci_alloc_irq_vectors(instance->pdev, 1, 1, PCI_IRQ_LEGACY); + i = pci_alloc_irq_vectors(instance->pdev, 1, 1, PCI_IRQ_INTX); if (i < 0) goto fail_init_adapter; } @@ -7844,7 +7844,7 @@ megasas_resume(struct device *dev) if (!instance->msix_vectors) { rval = pci_alloc_irq_vectors(instance->pdev, 1, 1, - PCI_IRQ_LEGACY); + PCI_IRQ_INTX); if (rval < 0) goto fail_reenable_msix; } diff --git a/drivers/scsi/mpi3mr/mpi3mr_transport.c b/drivers/scsi/mpi3mr/mpi3mr_transport.c index 3b1d02ae8551..329cc6ec3b58 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_transport.c +++ b/drivers/scsi/mpi3mr/mpi3mr_transport.c @@ -7,6 +7,8 @@ * */ +#include <linux/vmalloc.h> + #include "mpi3mr.h" /** diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index 105917ea70ff..258647fc6bdd 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -3515,7 +3515,7 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc) ioc_info(ioc, "High IOPs queues : disabled\n"); ioc->reply_queue_count = 1; ioc->iopoll_q_start_index = ioc->reply_queue_count - 0; - r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY); + r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_INTX); if (r < 0) { dfailprintk(ioc, ioc_info(ioc, "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n", diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c index 0efe2fc8b308..a2a084c8075e 100644 --- a/drivers/scsi/pmcraid.c +++ b/drivers/scsi/pmcraid.c @@ -4036,7 +4036,7 @@ static int pmcraid_register_interrupt_handler(struct pmcraid_instance *pinstance) { struct pci_dev *pdev = pinstance->pdev; - unsigned int irq_flag = PCI_IRQ_LEGACY, flag; + unsigned int irq_flag = PCI_IRQ_INTX, flag; int num_hrrq, rc, i; irq_handler_t isr; diff --git a/drivers/scsi/scsicam.c b/drivers/scsi/scsicam.c index e2c7d8ef205f..dd69342bbe78 100644 --- a/drivers/scsi/scsicam.c +++ b/drivers/scsi/scsicam.c @@ -32,7 +32,7 @@ */ unsigned char *scsi_bios_ptable(struct block_device *dev) { - struct address_space *mapping = bdev_whole(dev)->bd_inode->i_mapping; + struct address_space *mapping = bdev_whole(dev)->bd_mapping; unsigned char *res = NULL; struct folio *folio; diff --git a/drivers/scsi/vmw_pvscsi.c b/drivers/scsi/vmw_pvscsi.c index f88ecdb93a8a..c4fea077265e 100644 --- a/drivers/scsi/vmw_pvscsi.c +++ b/drivers/scsi/vmw_pvscsi.c @@ -1346,7 +1346,7 @@ exit: static int pvscsi_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - unsigned int irq_flag = PCI_IRQ_MSIX | PCI_IRQ_MSI | PCI_IRQ_LEGACY; + unsigned int irq_flag = PCI_IRQ_ALL_TYPES; struct pvscsi_adapter *adapter; struct pvscsi_adapter adapter_temp; struct Scsi_Host *host = NULL; diff --git a/drivers/soundwire/amd_init.c b/drivers/soundwire/amd_init.c index e45dc8261ab1..4cd26f3a21f5 100644 --- a/drivers/soundwire/amd_init.c +++ b/drivers/soundwire/amd_init.c @@ -17,42 +17,38 @@ #define ACP_PAD_PULLDOWN_CTRL 0x0001448 #define ACP_SW_PAD_KEEPER_EN 0x0001454 -#define AMD_SDW_PAD_PULLDOWN_CTRL_ENABLE_MASK 0x7f9a -#define AMD_SDW0_PAD_PULLDOWN_CTRL_ENABLE_MASK 0x7f9f -#define AMD_SDW1_PAD_PULLDOWN_CTRL_ENABLE_MASK 0x7ffa -#define AMD_SDW0_PAD_EN_MASK 1 -#define AMD_SDW1_PAD_EN_MASK 0x10 -#define AMD_SDW_PAD_EN_MASK (AMD_SDW0_PAD_EN_MASK | AMD_SDW1_PAD_EN_MASK) +#define AMD_SDW0_PAD_CTRL_MASK 0x60 +#define AMD_SDW1_PAD_CTRL_MASK 5 +#define AMD_SDW_PAD_CTRL_MASK (AMD_SDW0_PAD_CTRL_MASK | AMD_SDW1_PAD_CTRL_MASK) +#define AMD_SDW0_PAD_EN 1 +#define AMD_SDW1_PAD_EN 0x10 +#define AMD_SDW_PAD_EN (AMD_SDW0_PAD_EN | AMD_SDW1_PAD_EN) static int amd_enable_sdw_pads(void __iomem *mmio, u32 link_mask, struct device *dev) { - u32 val; - u32 pad_keeper_en_mask, pad_pulldown_ctrl_mask; + u32 pad_keeper_en, pad_pulldown_ctrl_mask; switch (link_mask) { case 1: - pad_keeper_en_mask = AMD_SDW0_PAD_EN_MASK; - pad_pulldown_ctrl_mask = AMD_SDW0_PAD_PULLDOWN_CTRL_ENABLE_MASK; + pad_keeper_en = AMD_SDW0_PAD_EN; + pad_pulldown_ctrl_mask = AMD_SDW0_PAD_CTRL_MASK; break; case 2: - pad_keeper_en_mask = AMD_SDW1_PAD_EN_MASK; - pad_pulldown_ctrl_mask = AMD_SDW1_PAD_PULLDOWN_CTRL_ENABLE_MASK; + pad_keeper_en = AMD_SDW1_PAD_EN; + pad_pulldown_ctrl_mask = AMD_SDW1_PAD_CTRL_MASK; break; case 3: - pad_keeper_en_mask = AMD_SDW_PAD_EN_MASK; - pad_pulldown_ctrl_mask = AMD_SDW_PAD_PULLDOWN_CTRL_ENABLE_MASK; + pad_keeper_en = AMD_SDW_PAD_EN; + pad_pulldown_ctrl_mask = AMD_SDW_PAD_CTRL_MASK; break; default: dev_err(dev, "No SDW Links are enabled\n"); return -ENODEV; } - val = readl(mmio + ACP_SW_PAD_KEEPER_EN); - val |= pad_keeper_en_mask; - writel(val, mmio + ACP_SW_PAD_KEEPER_EN); - val = readl(mmio + ACP_PAD_PULLDOWN_CTRL); - val &= pad_pulldown_ctrl_mask; - writel(val, mmio + ACP_PAD_PULLDOWN_CTRL); + amd_updatel(mmio, ACP_SW_PAD_KEEPER_EN, pad_keeper_en, pad_keeper_en); + amd_updatel(mmio, ACP_PAD_PULLDOWN_CTRL, pad_pulldown_ctrl_mask, 0); + return 0; } diff --git a/drivers/soundwire/amd_init.h b/drivers/soundwire/amd_init.h index 928b0c707162..5e7b43836a37 100644 --- a/drivers/soundwire/amd_init.h +++ b/drivers/soundwire/amd_init.h @@ -10,4 +10,12 @@ int amd_sdw_manager_start(struct amd_sdw_manager *amd_manager); +static inline void amd_updatel(void __iomem *mmio, int offset, u32 mask, u32 val) +{ + u32 tmp; + + tmp = readl(mmio + offset); + tmp = (tmp & ~mask) | val; + writel(tmp, mmio + offset); +} #endif diff --git a/drivers/soundwire/amd_manager.c b/drivers/soundwire/amd_manager.c index 6bcf8e75273c..20d94bcfc9b4 100644 --- a/drivers/soundwire/amd_manager.c +++ b/drivers/soundwire/amd_manager.c @@ -89,9 +89,8 @@ static void amd_enable_sdw_interrupts(struct amd_sdw_manager *amd_manager) u32 val; mutex_lock(amd_manager->acp_sdw_lock); - val = readl(amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); - val |= sdw_manager_reg_mask_array[amd_manager->instance]; - writel(val, amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); + val = sdw_manager_reg_mask_array[amd_manager->instance]; + amd_updatel(amd_manager->acp_mmio, ACP_EXTERNAL_INTR_CNTL(amd_manager->instance), val, val); mutex_unlock(amd_manager->acp_sdw_lock); writel(AMD_SDW_IRQ_MASK_0TO7, amd_manager->mmio + @@ -103,12 +102,12 @@ static void amd_enable_sdw_interrupts(struct amd_sdw_manager *amd_manager) static void amd_disable_sdw_interrupts(struct amd_sdw_manager *amd_manager) { - u32 val; + u32 irq_mask; mutex_lock(amd_manager->acp_sdw_lock); - val = readl(amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); - val &= ~sdw_manager_reg_mask_array[amd_manager->instance]; - writel(val, amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); + irq_mask = sdw_manager_reg_mask_array[amd_manager->instance]; + amd_updatel(amd_manager->acp_mmio, ACP_EXTERNAL_INTR_CNTL(amd_manager->instance), + irq_mask, 0); mutex_unlock(amd_manager->acp_sdw_lock); writel(0x00, amd_manager->mmio + ACP_SW_STATE_CHANGE_STATUS_MASK_0TO7); diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index f3fec15c3112..191e6cc6f962 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -1312,18 +1312,18 @@ static int sdw_slave_set_frequency(struct sdw_slave *slave) if (!(19200000 % mclk_freq)) { mclk_freq = 19200000; base = SDW_SCP_BASE_CLOCK_19200000_HZ; - } else if (!(24000000 % mclk_freq)) { - mclk_freq = 24000000; - base = SDW_SCP_BASE_CLOCK_24000000_HZ; - } else if (!(24576000 % mclk_freq)) { - mclk_freq = 24576000; - base = SDW_SCP_BASE_CLOCK_24576000_HZ; } else if (!(22579200 % mclk_freq)) { mclk_freq = 22579200; base = SDW_SCP_BASE_CLOCK_22579200_HZ; + } else if (!(24576000 % mclk_freq)) { + mclk_freq = 24576000; + base = SDW_SCP_BASE_CLOCK_24576000_HZ; } else if (!(32000000 % mclk_freq)) { mclk_freq = 32000000; base = SDW_SCP_BASE_CLOCK_32000000_HZ; + } else if (!(96000000 % mclk_freq)) { + mclk_freq = 24000000; + base = SDW_SCP_BASE_CLOCK_24000000_HZ; } else { dev_err(&slave->dev, "Unsupported clock base, mclk %d\n", @@ -1474,7 +1474,7 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) } do { - clear = status & ~SDW_DP0_INTERRUPTS; + clear = status & ~(SDW_DP0_INTERRUPTS | SDW_DP0_SDCA_CASCADE); if (status & SDW_DP0_INT_TEST_FAIL) { dev_err(&slave->dev, "Test fail for port 0\n"); diff --git a/drivers/soundwire/bus_type.c b/drivers/soundwire/bus_type.c index fd65b2360fc1..c32faace618f 100644 --- a/drivers/soundwire/bus_type.c +++ b/drivers/soundwire/bus_type.c @@ -126,8 +126,8 @@ static int sdw_drv_probe(struct device *dev) if (slave->prop.use_domain_irq) sdw_irq_create_mapping(slave); - /* init the sysfs as we have properties now */ - ret = sdw_slave_sysfs_init(slave); + /* init the dynamic sysfs attributes we need */ + ret = sdw_slave_sysfs_dpn_init(slave); if (ret < 0) dev_warn(dev, "Slave sysfs init failed:%d\n", ret); @@ -221,6 +221,7 @@ int __sdw_register_driver(struct sdw_driver *drv, struct module *owner) drv->driver.probe = sdw_drv_probe; drv->driver.remove = sdw_drv_remove; drv->driver.shutdown = sdw_drv_shutdown; + drv->driver.dev_groups = sdw_attr_groups; return driver_register(&drv->driver); } diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index 0efc1c3bee5f..74da99034dab 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -1236,7 +1236,7 @@ EXPORT_SYMBOL(sdw_cdns_enable_interrupt); static int cdns_allocate_pdi(struct sdw_cdns *cdns, struct sdw_cdns_pdi **stream, - u32 num, u32 pdi_offset) + u32 num) { struct sdw_cdns_pdi *pdi; int i; @@ -1249,7 +1249,7 @@ static int cdns_allocate_pdi(struct sdw_cdns *cdns, return -ENOMEM; for (i = 0; i < num; i++) { - pdi[i].num = i + pdi_offset; + pdi[i].num = i; } *stream = pdi; @@ -1266,7 +1266,6 @@ int sdw_cdns_pdi_init(struct sdw_cdns *cdns, struct sdw_cdns_stream_config config) { struct sdw_cdns_streams *stream; - int offset; int ret; cdns->pcm.num_bd = config.pcm_bd; @@ -1277,24 +1276,15 @@ int sdw_cdns_pdi_init(struct sdw_cdns *cdns, stream = &cdns->pcm; /* we allocate PDI0 and PDI1 which are used for Bulk */ - offset = 0; - - ret = cdns_allocate_pdi(cdns, &stream->bd, - stream->num_bd, offset); + ret = cdns_allocate_pdi(cdns, &stream->bd, stream->num_bd); if (ret) return ret; - offset += stream->num_bd; - - ret = cdns_allocate_pdi(cdns, &stream->in, - stream->num_in, offset); + ret = cdns_allocate_pdi(cdns, &stream->in, stream->num_in); if (ret) return ret; - offset += stream->num_in; - - ret = cdns_allocate_pdi(cdns, &stream->out, - stream->num_out, offset); + ret = cdns_allocate_pdi(cdns, &stream->out, stream->num_out); if (ret) return ret; @@ -1329,6 +1319,12 @@ static void cdns_init_clock_ctrl(struct sdw_cdns *cdns) u32 ssp_interval; int divider; + dev_dbg(cdns->dev, "mclk %d max %d row %d col %d\n", + prop->mclk_freq, + prop->max_clk_freq, + prop->default_row, + prop->default_col); + /* Set clock divider */ divider = (prop->mclk_freq / prop->max_clk_freq) - 1; @@ -1802,7 +1798,6 @@ EXPORT_SYMBOL(cdns_set_sdw_stream); * cdns_find_pdi() - Find a free PDI * * @cdns: Cadence instance - * @offset: Starting offset * @num: Number of PDIs * @pdi: PDI instances * @dai_id: DAI id @@ -1811,14 +1806,13 @@ EXPORT_SYMBOL(cdns_set_sdw_stream); * expected to match, return NULL otherwise. */ static struct sdw_cdns_pdi *cdns_find_pdi(struct sdw_cdns *cdns, - unsigned int offset, unsigned int num, struct sdw_cdns_pdi *pdi, int dai_id) { int i; - for (i = offset; i < offset + num; i++) + for (i = 0; i < num; i++) if (pdi[i].num == dai_id) return &pdi[i]; @@ -1872,15 +1866,15 @@ struct sdw_cdns_pdi *sdw_cdns_alloc_pdi(struct sdw_cdns *cdns, struct sdw_cdns_pdi *pdi = NULL; if (dir == SDW_DATA_DIR_RX) - pdi = cdns_find_pdi(cdns, 0, stream->num_in, stream->in, + pdi = cdns_find_pdi(cdns, stream->num_in, stream->in, dai_id); else - pdi = cdns_find_pdi(cdns, 0, stream->num_out, stream->out, + pdi = cdns_find_pdi(cdns, stream->num_out, stream->out, dai_id); /* check if we found a PDI, else find in bi-directional */ if (!pdi) - pdi = cdns_find_pdi(cdns, 2, stream->num_bd, stream->bd, + pdi = cdns_find_pdi(cdns, stream->num_bd, stream->bd, dai_id); if (pdi) { diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 1287a325c435..01e1a0f3ec39 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -345,8 +345,10 @@ static int intel_link_power_up(struct sdw_intel *sdw) u32 spa_mask, cpa_mask; u32 link_control; int ret = 0; + u32 clock_source; u32 syncprd; u32 sync_reg; + bool lcap_mlcs; mutex_lock(sdw->link_res->shim_lock); @@ -358,12 +360,35 @@ static int intel_link_power_up(struct sdw_intel *sdw) * is only dependent on the oscillator clock provided to * the IP, so adjust based on _DSD properties reported in DSDT * tables. The values reported are based on either 24MHz - * (CNL/CML) or 38.4 MHz (ICL/TGL+). + * (CNL/CML) or 38.4 MHz (ICL/TGL+). On MeteorLake additional + * frequencies are available with the MLCS clock source selection. */ - if (prop->mclk_freq % 6000000) - syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_38_4; - else - syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_24; + lcap_mlcs = intel_readl(shim, SDW_SHIM_LCAP) & SDW_SHIM_LCAP_MLCS_MASK; + + if (prop->mclk_freq % 6000000) { + if (prop->mclk_freq % 2400000) { + if (lcap_mlcs) { + syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_24_576; + clock_source = SDW_SHIM_MLCS_CARDINAL_CLK; + } else { + dev_err(sdw->cdns.dev, "%s: invalid clock configuration, mclk %d lcap_mlcs %d\n", + __func__, prop->mclk_freq, lcap_mlcs); + ret = -EINVAL; + goto out; + } + } else { + syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_38_4; + clock_source = SDW_SHIM_MLCS_XTAL_CLK; + } + } else { + if (lcap_mlcs) { + syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_96; + clock_source = SDW_SHIM_MLCS_AUDIO_PLL_CLK; + } else { + syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_24; + clock_source = SDW_SHIM_MLCS_XTAL_CLK; + } + } if (!*shim_mask) { dev_dbg(sdw->cdns.dev, "powering up all links\n"); @@ -403,6 +428,13 @@ static int intel_link_power_up(struct sdw_intel *sdw) "Failed to set SHIM_SYNC: %d\n", ret); goto out; } + + /* update link clock if needed */ + if (lcap_mlcs) { + link_control = intel_readl(shim, SDW_SHIM_LCTL); + u32p_replace_bits(&link_control, clock_source, SDW_SHIM_LCTL_MLCS_MASK); + intel_writel(shim, SDW_SHIM_LCTL, link_control); + } } *shim_mask |= BIT(link_id); @@ -668,6 +700,24 @@ static int intel_params_stream(struct sdw_intel *sdw, * DAI routines */ +static int intel_free_stream(struct sdw_intel *sdw, + struct snd_pcm_substream *substream, + struct snd_soc_dai *dai, + int link_id) +{ + struct sdw_intel_link_res *res = sdw->link_res; + struct sdw_intel_stream_free_data free_data; + + free_data.substream = substream; + free_data.dai = dai; + free_data.link_id = link_id; + + if (res->ops && res->ops->free_stream && res->dev) + return res->ops->free_stream(res->dev, &free_data); + + return 0; +} + static int intel_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct snd_soc_dai *dai) @@ -799,6 +849,7 @@ static int intel_hw_free(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct sdw_cdns *cdns = snd_soc_dai_get_drvdata(dai); + struct sdw_intel *sdw = cdns_to_intel(cdns); struct sdw_cdns_dai_runtime *dai_runtime; int ret; @@ -819,6 +870,12 @@ intel_hw_free(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) return ret; } + ret = intel_free_stream(sdw, substream, dai, sdw->instance); + if (ret < 0) { + dev_err(dai->dev, "intel_free_stream: failed %d\n", ret); + return ret; + } + dai_runtime->pdi = NULL; return 0; @@ -1062,4 +1119,3 @@ const struct sdw_intel_hw_ops sdw_intel_cnl_hw_ops = { .sync_check_cmdsync_unlocked = intel_check_cmdsync_unlocked, }; EXPORT_SYMBOL_NS(sdw_intel_cnl_hw_ops, SOUNDWIRE_INTEL); - diff --git a/drivers/soundwire/intel.h b/drivers/soundwire/intel.h index 511932c55216..b68e74c294e7 100644 --- a/drivers/soundwire/intel.h +++ b/drivers/soundwire/intel.h @@ -58,6 +58,13 @@ struct sdw_intel { #endif }; +struct sdw_intel_prop { + u16 doaise; + u16 doais; + u16 dodse; + u16 dods; +}; + enum intel_pdi_type { INTEL_PDI_IN = 0, INTEL_PDI_OUT = 1, diff --git a/drivers/soundwire/intel_ace2x.c b/drivers/soundwire/intel_ace2x.c index 8280baa3254b..8b1b6ad420cf 100644 --- a/drivers/soundwire/intel_ace2x.c +++ b/drivers/soundwire/intel_ace2x.c @@ -10,8 +10,10 @@ #include <linux/soundwire/sdw_registers.h> #include <linux/soundwire/sdw.h> #include <linux/soundwire/sdw_intel.h> -#include <sound/pcm_params.h> +#include <sound/hdaudio.h> #include <sound/hda-mlink.h> +#include <sound/hda_register.h> +#include <sound/pcm_params.h> #include "cadence_master.h" #include "bus.h" #include "intel.h" @@ -23,49 +25,86 @@ static void intel_shim_vs_init(struct sdw_intel *sdw) { void __iomem *shim_vs = sdw->link_res->shim_vs; + struct sdw_bus *bus = &sdw->cdns.bus; + struct sdw_intel_prop *intel_prop; + u16 doaise; + u16 doais; + u16 dodse; + u16 dods; u16 act; + intel_prop = bus->vendor_specific_prop; + doaise = intel_prop->doaise; + doais = intel_prop->doais; + dodse = intel_prop->dodse; + dods = intel_prop->dods; + act = intel_readw(shim_vs, SDW_SHIM2_INTEL_VS_ACTMCTL); - u16p_replace_bits(&act, 0x1, SDW_SHIM2_INTEL_VS_ACTMCTL_DOAIS); + u16p_replace_bits(&act, doaise, SDW_SHIM2_INTEL_VS_ACTMCTL_DOAISE); + u16p_replace_bits(&act, doais, SDW_SHIM2_INTEL_VS_ACTMCTL_DOAIS); + u16p_replace_bits(&act, dodse, SDW_SHIM2_INTEL_VS_ACTMCTL_DODSE); + u16p_replace_bits(&act, dods, SDW_SHIM2_INTEL_VS_ACTMCTL_DODS); act |= SDW_SHIM2_INTEL_VS_ACTMCTL_DACTQE; - act |= SDW_SHIM2_INTEL_VS_ACTMCTL_DODS; intel_writew(shim_vs, SDW_SHIM2_INTEL_VS_ACTMCTL, act); usleep_range(10, 15); } -static int intel_shim_check_wake(struct sdw_intel *sdw) +static void intel_shim_vs_set_clock_source(struct sdw_intel *sdw, u32 source) { - void __iomem *shim_vs; - u16 wake_sts; + void __iomem *shim_vs = sdw->link_res->shim_vs; + u32 val; - shim_vs = sdw->link_res->shim_vs; - wake_sts = intel_readw(shim_vs, SDW_SHIM2_INTEL_VS_WAKESTS); + val = intel_readl(shim_vs, SDW_SHIM2_INTEL_VS_LVSCTL); - return wake_sts & SDW_SHIM2_INTEL_VS_WAKEEN_PWS; + u32p_replace_bits(&val, source, SDW_SHIM2_INTEL_VS_LVSCTL_MLCS); + + intel_writel(shim_vs, SDW_SHIM2_INTEL_VS_LVSCTL, val); + + dev_dbg(sdw->cdns.dev, "clock source %d LVSCTL %#x\n", source, val); +} + +static int intel_shim_check_wake(struct sdw_intel *sdw) +{ + /* + * We follow the HDaudio example and resume unconditionally + * without checking the WAKESTS bit for that specific link + */ + + return 1; } static void intel_shim_wake(struct sdw_intel *sdw, bool wake_enable) { - void __iomem *shim_vs = sdw->link_res->shim_vs; + u16 lsdiid = 0; u16 wake_en; u16 wake_sts; + int ret; + + mutex_lock(sdw->link_res->shim_lock); + + ret = hdac_bus_eml_sdw_get_lsdiid_unlocked(sdw->link_res->hbus, sdw->instance, &lsdiid); + if (ret < 0) + goto unlock; - wake_en = intel_readw(shim_vs, SDW_SHIM2_INTEL_VS_WAKEEN); + wake_en = snd_hdac_chip_readw(sdw->link_res->hbus, WAKEEN); if (wake_enable) { /* Enable the wakeup */ - wake_en |= SDW_SHIM2_INTEL_VS_WAKEEN_PWE; - intel_writew(shim_vs, SDW_SHIM2_INTEL_VS_WAKEEN, wake_en); + wake_en |= lsdiid; + + snd_hdac_chip_writew(sdw->link_res->hbus, WAKEEN, wake_en); } else { /* Disable the wake up interrupt */ - wake_en &= ~SDW_SHIM2_INTEL_VS_WAKEEN_PWE; - intel_writew(shim_vs, SDW_SHIM2_INTEL_VS_WAKEEN, wake_en); + wake_en &= ~lsdiid; + snd_hdac_chip_writew(sdw->link_res->hbus, WAKEEN, wake_en); /* Clear wake status (W1C) */ - wake_sts = intel_readw(shim_vs, SDW_SHIM2_INTEL_VS_WAKESTS); - wake_sts |= SDW_SHIM2_INTEL_VS_WAKEEN_PWS; - intel_writew(shim_vs, SDW_SHIM2_INTEL_VS_WAKESTS, wake_sts); + wake_sts = snd_hdac_chip_readw(sdw->link_res->hbus, STATESTS); + wake_sts |= lsdiid; + snd_hdac_chip_writew(sdw->link_res->hbus, STATESTS, wake_sts); } +unlock: + mutex_unlock(sdw->link_res->shim_lock); } static int intel_link_power_up(struct sdw_intel *sdw) @@ -74,36 +113,45 @@ static int intel_link_power_up(struct sdw_intel *sdw) struct sdw_master_prop *prop = &bus->prop; u32 *shim_mask = sdw->link_res->shim_mask; unsigned int link_id = sdw->instance; + u32 clock_source; u32 syncprd; int ret; + if (prop->mclk_freq % 6000000) { + if (prop->mclk_freq % 2400000) { + syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_24_576; + clock_source = SDW_SHIM2_MLCS_CARDINAL_CLK; + } else { + syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_38_4; + clock_source = SDW_SHIM2_MLCS_XTAL_CLK; + } + } else { + syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_96; + clock_source = SDW_SHIM2_MLCS_AUDIO_PLL_CLK; + } + mutex_lock(sdw->link_res->shim_lock); + ret = hdac_bus_eml_sdw_power_up_unlocked(sdw->link_res->hbus, link_id); + if (ret < 0) { + dev_err(sdw->cdns.dev, "%s: hdac_bus_eml_sdw_power_up failed: %d\n", + __func__, ret); + goto out; + } + + intel_shim_vs_set_clock_source(sdw, clock_source); + if (!*shim_mask) { /* we first need to program the SyncPRD/CPU registers */ dev_dbg(sdw->cdns.dev, "first link up, programming SYNCPRD\n"); - if (prop->mclk_freq % 6000000) - syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_38_4; - else - syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_24; - ret = hdac_bus_eml_sdw_set_syncprd_unlocked(sdw->link_res->hbus, syncprd); if (ret < 0) { dev_err(sdw->cdns.dev, "%s: hdac_bus_eml_sdw_set_syncprd failed: %d\n", __func__, ret); goto out; } - } - - ret = hdac_bus_eml_sdw_power_up_unlocked(sdw->link_res->hbus, link_id); - if (ret < 0) { - dev_err(sdw->cdns.dev, "%s: hdac_bus_eml_sdw_power_up failed: %d\n", - __func__, ret); - goto out; - } - if (!*shim_mask) { /* SYNCPU will change once link is active */ ret = hdac_bus_eml_sdw_wait_syncpu_unlocked(sdw->link_res->hbus); if (ret < 0) { @@ -268,6 +316,11 @@ static int intel_hw_params(struct snd_pcm_substream *substream, goto error; } + /* use same definitions for alh_id as previous generations */ + pdi->intel_alh_id = (sdw->instance * 16) + pdi->num + 3; + if (pdi->num >= 2) + pdi->intel_alh_id += 2; + /* the SHIM will be configured in the callback functions */ sdw_cdns_config_stream(cdns, ch, dir, pdi); diff --git a/drivers/soundwire/intel_auxdevice.c b/drivers/soundwire/intel_auxdevice.c index 95125cc2fc59..17cf27e6ea73 100644 --- a/drivers/soundwire/intel_auxdevice.c +++ b/drivers/soundwire/intel_auxdevice.c @@ -122,6 +122,7 @@ static void generic_new_peripheral_assigned(struct sdw_bus *bus, static int sdw_master_read_intel_prop(struct sdw_bus *bus) { struct sdw_master_prop *prop = &bus->prop; + struct sdw_intel_prop *intel_prop; struct fwnode_handle *link; char name[32]; u32 quirk_mask; @@ -153,6 +154,36 @@ static int sdw_master_read_intel_prop(struct sdw_bus *bus) prop->quirks = SDW_MASTER_QUIRKS_CLEAR_INITIAL_CLASH | SDW_MASTER_QUIRKS_CLEAR_INITIAL_PARITY; + intel_prop = devm_kzalloc(bus->dev, sizeof(*intel_prop), GFP_KERNEL); + if (!intel_prop) + return -ENOMEM; + + /* initialize with hardware defaults, in case the properties are not found */ + intel_prop->doaise = 0x1; + intel_prop->doais = 0x3; + intel_prop->dodse = 0x0; + intel_prop->dods = 0x1; + + fwnode_property_read_u16(link, + "intel-sdw-doaise", + &intel_prop->doaise); + fwnode_property_read_u16(link, + "intel-sdw-doais", + &intel_prop->doais); + fwnode_property_read_u16(link, + "intel-sdw-dodse", + &intel_prop->dodse); + fwnode_property_read_u16(link, + "intel-sdw-dods", + &intel_prop->dods); + bus->vendor_specific_prop = intel_prop; + + dev_dbg(bus->dev, "doaise %#x doais %#x dodse %#x dods %#x\n", + intel_prop->doaise, + intel_prop->doais, + intel_prop->dodse, + intel_prop->dods); + return 0; } @@ -440,7 +471,7 @@ int intel_link_process_wakeen_event(struct auxiliary_device *auxdev) * PM calls */ -static int intel_resume_child_device(struct device *dev, void *data) +int intel_resume_child_device(struct device *dev, void *data) { int ret; struct sdw_slave *slave = dev_to_sdw_dev(dev); @@ -454,9 +485,9 @@ static int intel_resume_child_device(struct device *dev, void *data) return 0; } - ret = pm_request_resume(dev); + ret = pm_runtime_resume(dev); if (ret < 0) { - dev_err(dev, "%s: pm_request_resume failed: %d\n", __func__, ret); + dev_err(dev, "%s: pm_runtime_resume failed: %d\n", __func__, ret); return ret; } @@ -499,9 +530,9 @@ static int __maybe_unused intel_pm_prepare(struct device *dev) * first resume the device for this link. This will also by construction * resume the PCI parent device. */ - ret = pm_request_resume(dev); + ret = pm_runtime_resume(dev); if (ret < 0) { - dev_err(dev, "%s: pm_request_resume failed: %d\n", __func__, ret); + dev_err(dev, "%s: pm_runtime_resume failed: %d\n", __func__, ret); return 0; } diff --git a/drivers/soundwire/intel_auxdevice.h b/drivers/soundwire/intel_auxdevice.h index a00ecde95563..acaae0bd6592 100644 --- a/drivers/soundwire/intel_auxdevice.h +++ b/drivers/soundwire/intel_auxdevice.h @@ -6,6 +6,7 @@ int intel_link_startup(struct auxiliary_device *auxdev); int intel_link_process_wakeen_event(struct auxiliary_device *auxdev); +int intel_resume_child_device(struct device *dev, void *data); struct sdw_intel_link_dev { struct auxiliary_device auxdev; diff --git a/drivers/soundwire/intel_init.c b/drivers/soundwire/intel_init.c index 534c8795e7e8..a09134b97cd6 100644 --- a/drivers/soundwire/intel_init.c +++ b/drivers/soundwire/intel_init.c @@ -16,6 +16,7 @@ #include <linux/pm_runtime.h> #include <linux/soundwire/sdw_intel.h> #include "cadence_master.h" +#include "bus.h" #include "intel.h" #include "intel_auxdevice.h" @@ -356,6 +357,19 @@ EXPORT_SYMBOL_NS(sdw_intel_startup, SOUNDWIRE_INTEL_INIT); */ void sdw_intel_exit(struct sdw_intel_ctx *ctx) { + struct sdw_intel_link_res *link; + + /* we first resume links and devices and wait synchronously before the cleanup */ + list_for_each_entry(link, &ctx->link_list, list) { + struct sdw_bus *bus = &link->cdns->bus; + int ret; + + ret = device_for_each_child(bus->dev, NULL, intel_resume_child_device); + if (ret < 0) + dev_err(bus->dev, "%s: intel_resume_child_device failed: %d\n", + __func__, ret); + } + sdw_intel_cleanup(ctx); kfree(ctx->ids); kfree(ctx->ldev); diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c index 3c4d6debab1f..ce5cf3ecceb5 100644 --- a/drivers/soundwire/qcom.c +++ b/drivers/soundwire/qcom.c @@ -905,6 +905,18 @@ static int qcom_swrm_init(struct qcom_swrm_ctrl *ctrl) return 0; } +static int qcom_swrm_read_prop(struct sdw_bus *bus) +{ + struct qcom_swrm_ctrl *ctrl = to_qcom_sdw(bus); + + if (ctrl->version >= SWRM_VERSION_2_0_0) { + bus->multi_link = true; + bus->hw_sync_min_links = 3; + } + + return 0; +} + static enum sdw_command_response qcom_swrm_xfer_msg(struct sdw_bus *bus, struct sdw_msg *msg) { @@ -1056,6 +1068,7 @@ static const struct sdw_master_port_ops qcom_swrm_port_ops = { }; static const struct sdw_master_ops qcom_swrm_ops = { + .read_prop = qcom_swrm_read_prop, .xfer_msg = qcom_swrm_xfer_msg, .pre_bank_switch = qcom_swrm_pre_bank_switch, }; @@ -1173,6 +1186,15 @@ static int qcom_swrm_stream_alloc_ports(struct qcom_swrm_ctrl *ctrl, mutex_lock(&ctrl->port_lock); list_for_each_entry(m_rt, &stream->master_list, stream_node) { + /* + * For streams with multiple masters: + * Allocate ports only for devices connected to this master. + * Such devices will have ports allocated by their own master + * and its qcom_swrm_stream_alloc_ports() call. + */ + if (ctrl->bus.id != m_rt->bus->id) + continue; + if (m_rt->direction == SDW_DATA_DIR_RX) { maxport = ctrl->num_dout_ports; port_mask = &ctrl->dout_port_mask; @@ -1636,14 +1658,12 @@ err_init: return ret; } -static int qcom_swrm_remove(struct platform_device *pdev) +static void qcom_swrm_remove(struct platform_device *pdev) { struct qcom_swrm_ctrl *ctrl = dev_get_drvdata(&pdev->dev); sdw_bus_master_delete(&ctrl->bus); clk_disable_unprepare(ctrl->hclk); - - return 0; } static int __maybe_unused swrm_runtime_resume(struct device *dev) @@ -1769,7 +1789,7 @@ MODULE_DEVICE_TABLE(of, qcom_swrm_of_match); static struct platform_driver qcom_swrm_driver = { .probe = &qcom_swrm_probe, - .remove = &qcom_swrm_remove, + .remove_new = qcom_swrm_remove, .driver = { .name = "qcom-soundwire", .of_match_table = qcom_swrm_of_match, diff --git a/drivers/soundwire/sysfs_local.h b/drivers/soundwire/sysfs_local.h index 7268bc24c538..fa048e112629 100644 --- a/drivers/soundwire/sysfs_local.h +++ b/drivers/soundwire/sysfs_local.h @@ -11,8 +11,10 @@ /* basic attributes to report status of Slave (attachment, dev_num) */ extern const struct attribute_group *sdw_slave_status_attr_groups[]; +/* attributes for all soundwire devices */ +extern const struct attribute_group *sdw_attr_groups[]; + /* additional device-managed properties reported after driver probe */ -int sdw_slave_sysfs_init(struct sdw_slave *slave); int sdw_slave_sysfs_dpn_init(struct sdw_slave *slave); #endif /* __SDW_SYSFS_LOCAL_H */ diff --git a/drivers/soundwire/sysfs_slave.c b/drivers/soundwire/sysfs_slave.c index 3210359cd944..f4259710dd0f 100644 --- a/drivers/soundwire/sysfs_slave.c +++ b/drivers/soundwire/sysfs_slave.c @@ -105,7 +105,10 @@ static struct attribute *slave_attrs[] = { &dev_attr_modalias.attr, NULL, }; -ATTRIBUTE_GROUPS(slave); + +static const struct attribute_group slave_attr_group = { + .attrs = slave_attrs, +}; static struct attribute *slave_dev_attrs[] = { &dev_attr_mipi_revision.attr, @@ -126,10 +129,6 @@ static struct attribute *slave_dev_attrs[] = { NULL, }; -/* - * we don't use ATTRIBUTES_GROUP here since we want to add a subdirectory - * for device-level properties - */ static const struct attribute_group sdw_slave_dev_attr_group = { .attrs = slave_dev_attrs, .name = "dev-properties", @@ -181,41 +180,38 @@ static struct attribute *dp0_attrs[] = { NULL, }; -/* - * we don't use ATTRIBUTES_GROUP here since we want to add a subdirectory - * for dp0-level properties - */ -static const struct attribute_group dp0_group = { - .attrs = dp0_attrs, - .name = "dp0", -}; - -int sdw_slave_sysfs_init(struct sdw_slave *slave) +static umode_t dp0_attr_visible(struct kobject *kobj, struct attribute *attr, + int n) { - int ret; + struct sdw_slave *slave = dev_to_sdw_dev(kobj_to_dev(kobj)); - ret = devm_device_add_groups(&slave->dev, slave_groups); - if (ret < 0) - return ret; + if (slave->prop.dp0_prop) + return attr->mode; + return 0; +} - ret = devm_device_add_group(&slave->dev, &sdw_slave_dev_attr_group); - if (ret < 0) - return ret; +static bool dp0_group_visible(struct kobject *kobj) +{ + struct sdw_slave *slave = dev_to_sdw_dev(kobj_to_dev(kobj)); - if (slave->prop.dp0_prop) { - ret = devm_device_add_group(&slave->dev, &dp0_group); - if (ret < 0) - return ret; - } + if (slave->prop.dp0_prop) + return true; + return false; +} +DEFINE_SYSFS_GROUP_VISIBLE(dp0); - if (slave->prop.source_ports || slave->prop.sink_ports) { - ret = sdw_slave_sysfs_dpn_init(slave); - if (ret < 0) - return ret; - } +static const struct attribute_group dp0_group = { + .attrs = dp0_attrs, + .is_visible = SYSFS_GROUP_VISIBLE(dp0), + .name = "dp0", +}; - return 0; -} +const struct attribute_group *sdw_attr_groups[] = { + &slave_attr_group, + &sdw_slave_dev_attr_group, + &dp0_group, + NULL, +}; /* * the status is shown in capital letters for UNATTACHED and RESERVED diff --git a/drivers/soundwire/sysfs_slave_dpn.c b/drivers/soundwire/sysfs_slave_dpn.c index c4b6543c09fd..a3fb380ee519 100644 --- a/drivers/soundwire/sysfs_slave_dpn.c +++ b/drivers/soundwire/sysfs_slave_dpn.c @@ -283,6 +283,9 @@ int sdw_slave_sysfs_dpn_init(struct sdw_slave *slave) int ret; int i; + if (!slave->prop.source_ports && !slave->prop.sink_ports) + return 0; + mask = slave->prop.source_ports; for_each_set_bit(i, &mask, 32) { ret = add_all_attributes(&slave->dev, i, 1); diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm.c b/drivers/staging/media/atomisp/pci/hmm/hmm.c index bb12644fd033..3e2899ad8517 100644 --- a/drivers/staging/media/atomisp/pci/hmm/hmm.c +++ b/drivers/staging/media/atomisp/pci/hmm/hmm.c @@ -205,7 +205,7 @@ static ia_css_ptr __hmm_alloc(size_t bytes, enum hmm_bo_type type, } dev_dbg(atomisp_dev, "pages: 0x%08x (%zu bytes), type: %d, vmalloc %p\n", - bo->start, bytes, type, vmalloc); + bo->start, bytes, type, vmalloc_noprof); return bo->start; diff --git a/drivers/staging/rtl8723bs/Makefile b/drivers/staging/rtl8723bs/Makefile index 590bde02058c..7f5067e89295 100644 --- a/drivers/staging/rtl8723bs/Makefile +++ b/drivers/staging/rtl8723bs/Makefile @@ -62,4 +62,4 @@ r8723bs-y = \ obj-$(CONFIG_RTL8723BS) := r8723bs.o -ccflags-y += -I$(srctree)/$(src)/include -I$(srctree)/$(src)/hal +ccflags-y += -I$(src)/include -I$(src)/hal diff --git a/drivers/thermal/mediatek/lvts_thermal.c b/drivers/thermal/mediatek/lvts_thermal.c index 86b2f44355ac..0bb3a495b56e 100644 --- a/drivers/thermal/mediatek/lvts_thermal.c +++ b/drivers/thermal/mediatek/lvts_thermal.c @@ -105,8 +105,6 @@ struct lvts_sensor_data { struct lvts_ctrl_data { struct lvts_sensor_data lvts_sensor[LVTS_SENSOR_MAX]; - int cal_offset[LVTS_SENSOR_MAX]; - int num_lvts_sensor; u8 valid_sensor_mask; int offset; int mode; @@ -118,9 +116,9 @@ struct lvts_ctrl_data { ((s2) ? BIT(2) : 0) | \ ((s3) ? BIT(3) : 0)) -#define lvts_for_each_valid_sensor(i, lvts_ctrl_data) \ +#define lvts_for_each_valid_sensor(i, lvts_ctrl) \ for ((i) = 0; (i) < LVTS_SENSOR_MAX; (i)++) \ - if (!((lvts_ctrl_data)->valid_sensor_mask & BIT(i))) \ + if (!((lvts_ctrl)->valid_sensor_mask & BIT(i))) \ continue; \ else @@ -147,6 +145,7 @@ struct lvts_ctrl { const struct lvts_data *lvts_data; u32 calibration[LVTS_SENSOR_MAX]; u32 hw_tshut_raw_temp; + u8 valid_sensor_mask; int mode; void __iomem *base; int low_thresh; @@ -358,7 +357,7 @@ static bool lvts_should_update_thresh(struct lvts_ctrl *lvts_ctrl, int high) if (high > lvts_ctrl->high_thresh) return true; - lvts_for_each_valid_sensor(i, lvts_ctrl->lvts_data->lvts_ctrl) + lvts_for_each_valid_sensor(i, lvts_ctrl) if (lvts_ctrl->sensors[i].high_thresh == lvts_ctrl->high_thresh && lvts_ctrl->sensors[i].low_thresh == lvts_ctrl->low_thresh) return false; @@ -619,6 +618,8 @@ static int lvts_sensor_init(struct device *dev, struct lvts_ctrl *lvts_ctrl, lvts_sensor[i].high_thresh = INT_MIN; }; + lvts_ctrl->valid_sensor_mask = lvts_ctrl_data->valid_sensor_mask; + return 0; } @@ -1114,7 +1115,7 @@ static int lvts_ctrl_start(struct device *dev, struct lvts_ctrl *lvts_ctrl) u32 *sensor_bitmap = lvts_ctrl->mode == LVTS_MSR_IMMEDIATE_MODE ? sensor_imm_bitmap : sensor_filt_bitmap; - lvts_for_each_valid_sensor(i, lvts_ctrl->lvts_data->lvts_ctrl) { + lvts_for_each_valid_sensor(i, lvts_ctrl) { int dt_id = lvts_sensors[i].dt_id; @@ -1271,6 +1272,8 @@ static int lvts_probe(struct platform_device *pdev) return -ENOMEM; lvts_data = of_device_get_match_data(dev); + if (!lvts_data) + return -ENODEV; lvts_td->clk = devm_clk_get_enabled(dev, NULL); if (IS_ERR(lvts_td->clk)) diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 11750a145d74..54cce4e523bc 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -1398,8 +1398,15 @@ thermal_zone_device_register_with_trips(const char *type, tz->device.class = thermal_class; tz->devdata = devdata; tz->num_trips = num_trips; - for_each_trip_desc(tz, td) + for_each_trip_desc(tz, td) { td->trip = *trip++; + /* + * Mark all thresholds as invalid to start with even though + * this only matters for the trips that start as invalid and + * become valid later. + */ + td->threshold = INT_MAX; + } thermal_set_delay_jiffies(&tz->passive_delay_jiffies, passive_delay); thermal_set_delay_jiffies(&tz->polling_delay_jiffies, polling_delay); diff --git a/drivers/thermal/thermal_trip.c b/drivers/thermal/thermal_trip.c index 21ece8399997..d6a6acc78ddb 100644 --- a/drivers/thermal/thermal_trip.c +++ b/drivers/thermal/thermal_trip.c @@ -152,6 +152,24 @@ void thermal_zone_set_trip_temp(struct thermal_zone_device *tz, if (trip->temperature == temp) return; + if (temp == THERMAL_TEMP_INVALID) { + struct thermal_trip_desc *td = trip_to_trip_desc(trip); + + if (trip->type == THERMAL_TRIP_PASSIVE && + tz->temperature >= td->threshold) { + /* + * The trip has been crossed, so the thermal zone's + * passive count needs to be adjusted. + */ + tz->passive--; + WARN_ON_ONCE(tz->passive < 0); + } + /* + * Invalidate the threshold to avoid triggering a spurious + * trip crossing notification when the trip becomes valid. + */ + td->threshold = INT_MAX; + } trip->temperature = temp; thermal_notify_tz_trip_change(tz, trip); } diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c index e2e4f99f9d34..40af74b55933 100644 --- a/drivers/tty/serial/8250/8250_pci.c +++ b/drivers/tty/serial/8250/8250_pci.c @@ -4108,7 +4108,7 @@ pciserial_init_ports(struct pci_dev *dev, const struct pciserial_board *board) rc = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_ALL_TYPES); } else { pci_dbg(dev, "Using legacy interrupts\n"); - rc = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_LEGACY); + rc = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_INTX); } if (rc < 0) { kfree(priv); diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c index 7ce7bb164005..58ea1e1391ce 100644 --- a/drivers/tty/serial/kgdboc.c +++ b/drivers/tty/serial/kgdboc.c @@ -19,6 +19,7 @@ #include <linux/console.h> #include <linux/vt_kern.h> #include <linux/input.h> +#include <linux/irq_work.h> #include <linux/module.h> #include <linux/platform_device.h> #include <linux/serial_core.h> @@ -48,6 +49,25 @@ static struct kgdb_io kgdboc_earlycon_io_ops; static int (*earlycon_orig_exit)(struct console *con); #endif /* IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE) */ +/* + * When we leave the debug trap handler we need to reset the keyboard status + * (since the original keyboard state gets partially clobbered by kdb use of + * the keyboard). + * + * The path to deliver the reset is somewhat circuitous. + * + * To deliver the reset we register an input handler, reset the keyboard and + * then deregister the input handler. However, to get this done right, we do + * have to carefully manage the calling context because we can only register + * input handlers from task context. + * + * In particular we need to trigger the action from the debug trap handler with + * all its NMI and/or NMI-like oddities. To solve this the kgdboc trap exit code + * (the "post_exception" callback) uses irq_work_queue(), which is NMI-safe, to + * schedule a callback from a hardirq context. From there we have to defer the + * work again, this time using schedule_work(), to get a callback using the + * system workqueue, which runs in task context. + */ #ifdef CONFIG_KDB_KEYBOARD static int kgdboc_reset_connect(struct input_handler *handler, struct input_dev *dev, @@ -99,10 +119,17 @@ static void kgdboc_restore_input_helper(struct work_struct *dummy) static DECLARE_WORK(kgdboc_restore_input_work, kgdboc_restore_input_helper); +static void kgdboc_queue_restore_input_helper(struct irq_work *unused) +{ + schedule_work(&kgdboc_restore_input_work); +} + +static DEFINE_IRQ_WORK(kgdboc_restore_input_irq_work, kgdboc_queue_restore_input_helper); + static void kgdboc_restore_input(void) { if (likely(system_state == SYSTEM_RUNNING)) - schedule_work(&kgdboc_restore_input_work); + irq_work_queue(&kgdboc_restore_input_irq_work); } static int kgdboc_register_kbd(char **cptr) @@ -133,6 +160,7 @@ static void kgdboc_unregister_kbd(void) i--; } } + irq_work_sync(&kgdboc_restore_input_irq_work); flush_work(&kgdboc_restore_input_work); } #else /* ! CONFIG_KDB_KEYBOARD */ diff --git a/drivers/usb/core/hcd-pci.c b/drivers/usb/core/hcd-pci.c index ee3156f49533..a08f3f228e6d 100644 --- a/drivers/usb/core/hcd-pci.c +++ b/drivers/usb/core/hcd-pci.c @@ -189,7 +189,8 @@ int usb_hcd_pci_probe(struct pci_dev *dev, const struct hc_driver *driver) * make sure irq setup is not touched for xhci in generic hcd code */ if ((driver->flags & HCD_MASK) < HCD_USB3) { - retval = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_LEGACY | PCI_IRQ_MSI); + retval = pci_alloc_irq_vectors(dev, 1, 1, + PCI_IRQ_INTX | PCI_IRQ_MSI); if (retval < 0) { dev_err(&dev->dev, "Found HC with no IRQ. Check BIOS/PCI %s setup!\n", diff --git a/drivers/vfio/cdx/Makefile b/drivers/vfio/cdx/Makefile index cd4a2e6fe609..df92b320122a 100644 --- a/drivers/vfio/cdx/Makefile +++ b/drivers/vfio/cdx/Makefile @@ -5,4 +5,4 @@ obj-$(CONFIG_VFIO_CDX) += vfio-cdx.o -vfio-cdx-objs := main.o +vfio-cdx-objs := main.o intr.o diff --git a/drivers/vfio/cdx/intr.c b/drivers/vfio/cdx/intr.c new file mode 100644 index 000000000000..986fa2a45fa4 --- /dev/null +++ b/drivers/vfio/cdx/intr.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022-2023, Advanced Micro Devices, Inc. + */ + +#include <linux/vfio.h> +#include <linux/slab.h> +#include <linux/types.h> +#include <linux/eventfd.h> +#include <linux/msi.h> +#include <linux/interrupt.h> + +#include "linux/cdx/cdx_bus.h" +#include "private.h" + +static irqreturn_t vfio_cdx_msihandler(int irq_no, void *arg) +{ + struct eventfd_ctx *trigger = arg; + + eventfd_signal(trigger); + return IRQ_HANDLED; +} + +static int vfio_cdx_msi_enable(struct vfio_cdx_device *vdev, int nvec) +{ + struct cdx_device *cdx_dev = to_cdx_device(vdev->vdev.dev); + struct device *dev = vdev->vdev.dev; + int msi_idx, ret; + + vdev->cdx_irqs = kcalloc(nvec, sizeof(struct vfio_cdx_irq), GFP_KERNEL); + if (!vdev->cdx_irqs) + return -ENOMEM; + + ret = cdx_enable_msi(cdx_dev); + if (ret) { + kfree(vdev->cdx_irqs); + return ret; + } + + /* Allocate cdx MSIs */ + ret = msi_domain_alloc_irqs(dev, MSI_DEFAULT_DOMAIN, nvec); + if (ret) { + cdx_disable_msi(cdx_dev); + kfree(vdev->cdx_irqs); + return ret; + } + + for (msi_idx = 0; msi_idx < nvec; msi_idx++) + vdev->cdx_irqs[msi_idx].irq_no = msi_get_virq(dev, msi_idx); + + vdev->msi_count = nvec; + vdev->config_msi = 1; + + return 0; +} + +static int vfio_cdx_msi_set_vector_signal(struct vfio_cdx_device *vdev, + int vector, int fd) +{ + struct eventfd_ctx *trigger; + int irq_no, ret; + + if (vector < 0 || vector >= vdev->msi_count) + return -EINVAL; + + irq_no = vdev->cdx_irqs[vector].irq_no; + + if (vdev->cdx_irqs[vector].trigger) { + free_irq(irq_no, vdev->cdx_irqs[vector].trigger); + kfree(vdev->cdx_irqs[vector].name); + eventfd_ctx_put(vdev->cdx_irqs[vector].trigger); + vdev->cdx_irqs[vector].trigger = NULL; + } + + if (fd < 0) + return 0; + + vdev->cdx_irqs[vector].name = kasprintf(GFP_KERNEL, "vfio-msi[%d](%s)", + vector, dev_name(vdev->vdev.dev)); + if (!vdev->cdx_irqs[vector].name) + return -ENOMEM; + + trigger = eventfd_ctx_fdget(fd); + if (IS_ERR(trigger)) { + kfree(vdev->cdx_irqs[vector].name); + return PTR_ERR(trigger); + } + + ret = request_irq(irq_no, vfio_cdx_msihandler, 0, + vdev->cdx_irqs[vector].name, trigger); + if (ret) { + kfree(vdev->cdx_irqs[vector].name); + eventfd_ctx_put(trigger); + return ret; + } + + vdev->cdx_irqs[vector].trigger = trigger; + + return 0; +} + +static int vfio_cdx_msi_set_block(struct vfio_cdx_device *vdev, + unsigned int start, unsigned int count, + int32_t *fds) +{ + int i, j, ret = 0; + + if (start >= vdev->msi_count || start + count > vdev->msi_count) + return -EINVAL; + + for (i = 0, j = start; i < count && !ret; i++, j++) { + int fd = fds ? fds[i] : -1; + + ret = vfio_cdx_msi_set_vector_signal(vdev, j, fd); + } + + if (ret) { + for (--j; j >= (int)start; j--) + vfio_cdx_msi_set_vector_signal(vdev, j, -1); + } + + return ret; +} + +static void vfio_cdx_msi_disable(struct vfio_cdx_device *vdev) +{ + struct cdx_device *cdx_dev = to_cdx_device(vdev->vdev.dev); + struct device *dev = vdev->vdev.dev; + + vfio_cdx_msi_set_block(vdev, 0, vdev->msi_count, NULL); + + if (!vdev->config_msi) + return; + + msi_domain_free_irqs_all(dev, MSI_DEFAULT_DOMAIN); + cdx_disable_msi(cdx_dev); + kfree(vdev->cdx_irqs); + + vdev->cdx_irqs = NULL; + vdev->msi_count = 0; + vdev->config_msi = 0; +} + +static int vfio_cdx_set_msi_trigger(struct vfio_cdx_device *vdev, + unsigned int index, unsigned int start, + unsigned int count, u32 flags, + void *data) +{ + struct cdx_device *cdx_dev = to_cdx_device(vdev->vdev.dev); + int i; + + if (start + count > cdx_dev->num_msi) + return -EINVAL; + + if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) { + vfio_cdx_msi_disable(vdev); + return 0; + } + + if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { + s32 *fds = data; + int ret; + + if (vdev->config_msi) + return vfio_cdx_msi_set_block(vdev, start, count, + fds); + ret = vfio_cdx_msi_enable(vdev, cdx_dev->num_msi); + if (ret) + return ret; + + ret = vfio_cdx_msi_set_block(vdev, start, count, fds); + if (ret) + vfio_cdx_msi_disable(vdev); + + return ret; + } + + for (i = start; i < start + count; i++) { + if (!vdev->cdx_irqs[i].trigger) + continue; + if (flags & VFIO_IRQ_SET_DATA_NONE) { + eventfd_signal(vdev->cdx_irqs[i].trigger); + } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { + u8 *bools = data; + + if (bools[i - start]) + eventfd_signal(vdev->cdx_irqs[i].trigger); + } + } + + return 0; +} + +int vfio_cdx_set_irqs_ioctl(struct vfio_cdx_device *vdev, + u32 flags, unsigned int index, + unsigned int start, unsigned int count, + void *data) +{ + if (flags & VFIO_IRQ_SET_ACTION_TRIGGER) + return vfio_cdx_set_msi_trigger(vdev, index, start, + count, flags, data); + else + return -EINVAL; +} + +/* Free All IRQs for the given device */ +void vfio_cdx_irqs_cleanup(struct vfio_cdx_device *vdev) +{ + /* + * Device does not support any interrupt or the interrupts + * were not configured + */ + if (!vdev->cdx_irqs) + return; + + vfio_cdx_set_msi_trigger(vdev, 0, 0, 0, VFIO_IRQ_SET_DATA_NONE, NULL); +} diff --git a/drivers/vfio/cdx/main.c b/drivers/vfio/cdx/main.c index 9cff8d75789e..67465fad5b4b 100644 --- a/drivers/vfio/cdx/main.c +++ b/drivers/vfio/cdx/main.c @@ -61,6 +61,7 @@ static void vfio_cdx_close_device(struct vfio_device *core_vdev) kfree(vdev->regions); cdx_dev_reset(core_vdev->dev); + vfio_cdx_irqs_cleanup(vdev); } static int vfio_cdx_bm_ctrl(struct vfio_device *core_vdev, u32 flags, @@ -123,7 +124,7 @@ static int vfio_cdx_ioctl_get_info(struct vfio_cdx_device *vdev, info.flags |= VFIO_DEVICE_FLAGS_RESET; info.num_regions = cdx_dev->res_count; - info.num_irqs = 0; + info.num_irqs = cdx_dev->num_msi ? 1 : 0; return copy_to_user(arg, &info, minsz) ? -EFAULT : 0; } @@ -152,6 +153,62 @@ static int vfio_cdx_ioctl_get_region_info(struct vfio_cdx_device *vdev, return copy_to_user(arg, &info, minsz) ? -EFAULT : 0; } +static int vfio_cdx_ioctl_get_irq_info(struct vfio_cdx_device *vdev, + struct vfio_irq_info __user *arg) +{ + unsigned long minsz = offsetofend(struct vfio_irq_info, count); + struct cdx_device *cdx_dev = to_cdx_device(vdev->vdev.dev); + struct vfio_irq_info info; + + if (copy_from_user(&info, arg, minsz)) + return -EFAULT; + + if (info.argsz < minsz) + return -EINVAL; + + if (info.index >= 1) + return -EINVAL; + + if (!cdx_dev->num_msi) + return -EINVAL; + + info.flags = VFIO_IRQ_INFO_EVENTFD | VFIO_IRQ_INFO_NORESIZE; + info.count = cdx_dev->num_msi; + + return copy_to_user(arg, &info, minsz) ? -EFAULT : 0; +} + +static int vfio_cdx_ioctl_set_irqs(struct vfio_cdx_device *vdev, + struct vfio_irq_set __user *arg) +{ + unsigned long minsz = offsetofend(struct vfio_irq_set, count); + struct cdx_device *cdx_dev = to_cdx_device(vdev->vdev.dev); + struct vfio_irq_set hdr; + size_t data_size = 0; + u8 *data = NULL; + int ret = 0; + + if (copy_from_user(&hdr, arg, minsz)) + return -EFAULT; + + ret = vfio_set_irqs_validate_and_prepare(&hdr, cdx_dev->num_msi, + 1, &data_size); + if (ret) + return ret; + + if (data_size) { + data = memdup_user(arg->data, data_size); + if (IS_ERR(data)) + return PTR_ERR(data); + } + + ret = vfio_cdx_set_irqs_ioctl(vdev, hdr.flags, hdr.index, + hdr.start, hdr.count, data); + kfree(data); + + return ret; +} + static long vfio_cdx_ioctl(struct vfio_device *core_vdev, unsigned int cmd, unsigned long arg) { @@ -164,6 +221,10 @@ static long vfio_cdx_ioctl(struct vfio_device *core_vdev, return vfio_cdx_ioctl_get_info(vdev, uarg); case VFIO_DEVICE_GET_REGION_INFO: return vfio_cdx_ioctl_get_region_info(vdev, uarg); + case VFIO_DEVICE_GET_IRQ_INFO: + return vfio_cdx_ioctl_get_irq_info(vdev, uarg); + case VFIO_DEVICE_SET_IRQS: + return vfio_cdx_ioctl_set_irqs(vdev, uarg); case VFIO_DEVICE_RESET: return cdx_dev_reset(core_vdev->dev); default: diff --git a/drivers/vfio/cdx/private.h b/drivers/vfio/cdx/private.h index 8e9d25913728..dc56729b3114 100644 --- a/drivers/vfio/cdx/private.h +++ b/drivers/vfio/cdx/private.h @@ -13,6 +13,14 @@ static inline u64 vfio_cdx_index_to_offset(u32 index) return ((u64)(index) << VFIO_CDX_OFFSET_SHIFT); } +struct vfio_cdx_irq { + u32 flags; + u32 count; + int irq_no; + struct eventfd_ctx *trigger; + char *name; +}; + struct vfio_cdx_region { u32 flags; u32 type; @@ -23,8 +31,18 @@ struct vfio_cdx_region { struct vfio_cdx_device { struct vfio_device vdev; struct vfio_cdx_region *regions; + struct vfio_cdx_irq *cdx_irqs; u32 flags; #define BME_SUPPORT BIT(0) + u32 msi_count; + u8 config_msi; }; +int vfio_cdx_set_irqs_ioctl(struct vfio_cdx_device *vdev, + u32 flags, unsigned int index, + unsigned int start, unsigned int count, + void *data); + +void vfio_cdx_irqs_cleanup(struct vfio_cdx_device *vdev); + #endif /* VFIO_CDX_PRIVATE_H */ diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index 15821a2d77d2..bf50ffa10bde 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -69,4 +69,6 @@ source "drivers/vfio/pci/virtio/Kconfig" source "drivers/vfio/pci/nvgrace-gpu/Kconfig" +source "drivers/vfio/pci/qat/Kconfig" + endmenu diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile index ce7a61f1d912..cf00c0a7e55c 100644 --- a/drivers/vfio/pci/Makefile +++ b/drivers/vfio/pci/Makefile @@ -17,3 +17,5 @@ obj-$(CONFIG_PDS_VFIO_PCI) += pds/ obj-$(CONFIG_VIRTIO_VFIO_PCI) += virtio/ obj-$(CONFIG_NVGRACE_GPU_VFIO_PCI) += nvgrace-gpu/ + +obj-$(CONFIG_QAT_VFIO_PCI) += qat/ diff --git a/drivers/vfio/pci/pds/dirty.c b/drivers/vfio/pci/pds/dirty.c index 68e8f006dfdb..c51f5e4c3dd6 100644 --- a/drivers/vfio/pci/pds/dirty.c +++ b/drivers/vfio/pci/pds/dirty.c @@ -3,6 +3,7 @@ #include <linux/interval_tree.h> #include <linux/vfio.h> +#include <linux/vmalloc.h> #include <linux/pds/pds_common.h> #include <linux/pds/pds_core_if.h> diff --git a/drivers/vfio/pci/qat/Kconfig b/drivers/vfio/pci/qat/Kconfig new file mode 100644 index 000000000000..bf52cfa4b595 --- /dev/null +++ b/drivers/vfio/pci/qat/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +config QAT_VFIO_PCI + tristate "VFIO support for QAT VF PCI devices" + select VFIO_PCI_CORE + depends on CRYPTO_DEV_QAT_4XXX + help + This provides migration support for Intel(R) QAT Virtual Function + using the VFIO framework. + + To compile this as a module, choose M here: the module + will be called qat_vfio_pci. If you don't know what to do here, + say N. diff --git a/drivers/vfio/pci/qat/Makefile b/drivers/vfio/pci/qat/Makefile new file mode 100644 index 000000000000..5fe5c4ec19d3 --- /dev/null +++ b/drivers/vfio/pci/qat/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_QAT_VFIO_PCI) += qat_vfio_pci.o +qat_vfio_pci-y := main.o diff --git a/drivers/vfio/pci/qat/main.c b/drivers/vfio/pci/qat/main.c new file mode 100644 index 000000000000..e36740a282e7 --- /dev/null +++ b/drivers/vfio/pci/qat/main.c @@ -0,0 +1,702 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ + +#include <linux/anon_inodes.h> +#include <linux/container_of.h> +#include <linux/device.h> +#include <linux/file.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/sizes.h> +#include <linux/types.h> +#include <linux/uaccess.h> +#include <linux/vfio_pci_core.h> +#include <linux/qat/qat_mig_dev.h> + +/* + * The migration data of each Intel QAT VF device is encapsulated into a + * 4096 bytes block. The data consists of two parts. + * The first is a pre-configured set of attributes of the VF being migrated, + * which are only set when it is created. This can be migrated during pre-copy + * stage and used for a device compatibility check. + * The second is the VF state. This includes the required MMIO regions and + * the shadow states maintained by the QAT PF driver. This part can only be + * saved when the VF is fully quiesced and be migrated during stop-copy stage. + * Both these 2 parts of data are saved in hierarchical structures including + * a preamble section and several raw state sections. + * When the pre-configured part of the migration data is fully retrieved from + * user space, the preamble section are used to validate the correctness of + * the data blocks and check the version compatibility. The raw state sections + * are then used to do a device compatibility check. + * When the device transits from RESUMING state, the VF states are extracted + * from the raw state sections of the VF state part of the migration data and + * then loaded into the device. + */ + +struct qat_vf_migration_file { + struct file *filp; + /* protects migration region context */ + struct mutex lock; + bool disabled; + struct qat_vf_core_device *qat_vdev; + ssize_t filled_size; +}; + +struct qat_vf_core_device { + struct vfio_pci_core_device core_device; + struct qat_mig_dev *mdev; + /* protects migration state */ + struct mutex state_mutex; + enum vfio_device_mig_state mig_state; + struct qat_vf_migration_file *resuming_migf; + struct qat_vf_migration_file *saving_migf; +}; + +static int qat_vf_pci_open_device(struct vfio_device *core_vdev) +{ + struct qat_vf_core_device *qat_vdev = + container_of(core_vdev, struct qat_vf_core_device, + core_device.vdev); + struct vfio_pci_core_device *vdev = &qat_vdev->core_device; + int ret; + + ret = vfio_pci_core_enable(vdev); + if (ret) + return ret; + + ret = qat_vfmig_open(qat_vdev->mdev); + if (ret) { + vfio_pci_core_disable(vdev); + return ret; + } + qat_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; + + vfio_pci_core_finish_enable(vdev); + + return 0; +} + +static void qat_vf_disable_fd(struct qat_vf_migration_file *migf) +{ + mutex_lock(&migf->lock); + migf->disabled = true; + migf->filp->f_pos = 0; + migf->filled_size = 0; + mutex_unlock(&migf->lock); +} + +static void qat_vf_disable_fds(struct qat_vf_core_device *qat_vdev) +{ + if (qat_vdev->resuming_migf) { + qat_vf_disable_fd(qat_vdev->resuming_migf); + fput(qat_vdev->resuming_migf->filp); + qat_vdev->resuming_migf = NULL; + } + + if (qat_vdev->saving_migf) { + qat_vf_disable_fd(qat_vdev->saving_migf); + fput(qat_vdev->saving_migf->filp); + qat_vdev->saving_migf = NULL; + } +} + +static void qat_vf_pci_close_device(struct vfio_device *core_vdev) +{ + struct qat_vf_core_device *qat_vdev = container_of(core_vdev, + struct qat_vf_core_device, core_device.vdev); + + qat_vfmig_close(qat_vdev->mdev); + qat_vf_disable_fds(qat_vdev); + vfio_pci_core_close_device(core_vdev); +} + +static long qat_vf_precopy_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + struct qat_vf_migration_file *migf = filp->private_data; + struct qat_vf_core_device *qat_vdev = migf->qat_vdev; + struct qat_mig_dev *mig_dev = qat_vdev->mdev; + struct vfio_precopy_info info; + loff_t *pos = &filp->f_pos; + unsigned long minsz; + int ret = 0; + + if (cmd != VFIO_MIG_GET_PRECOPY_INFO) + return -ENOTTY; + + minsz = offsetofend(struct vfio_precopy_info, dirty_bytes); + + if (copy_from_user(&info, (void __user *)arg, minsz)) + return -EFAULT; + if (info.argsz < minsz) + return -EINVAL; + + mutex_lock(&qat_vdev->state_mutex); + if (qat_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY && + qat_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY_P2P) { + mutex_unlock(&qat_vdev->state_mutex); + return -EINVAL; + } + + mutex_lock(&migf->lock); + if (migf->disabled) { + ret = -ENODEV; + goto out; + } + + if (*pos > mig_dev->setup_size) { + ret = -EINVAL; + goto out; + } + + info.dirty_bytes = 0; + info.initial_bytes = mig_dev->setup_size - *pos; + +out: + mutex_unlock(&migf->lock); + mutex_unlock(&qat_vdev->state_mutex); + if (ret) + return ret; + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; +} + +static ssize_t qat_vf_save_read(struct file *filp, char __user *buf, + size_t len, loff_t *pos) +{ + struct qat_vf_migration_file *migf = filp->private_data; + struct qat_mig_dev *mig_dev = migf->qat_vdev->mdev; + ssize_t done = 0; + loff_t *offs; + int ret; + + if (pos) + return -ESPIPE; + offs = &filp->f_pos; + + mutex_lock(&migf->lock); + if (*offs > migf->filled_size || *offs < 0) { + done = -EINVAL; + goto out_unlock; + } + + if (migf->disabled) { + done = -ENODEV; + goto out_unlock; + } + + len = min_t(size_t, migf->filled_size - *offs, len); + if (len) { + ret = copy_to_user(buf, mig_dev->state + *offs, len); + if (ret) { + done = -EFAULT; + goto out_unlock; + } + *offs += len; + done = len; + } + +out_unlock: + mutex_unlock(&migf->lock); + return done; +} + +static int qat_vf_release_file(struct inode *inode, struct file *filp) +{ + struct qat_vf_migration_file *migf = filp->private_data; + + qat_vf_disable_fd(migf); + mutex_destroy(&migf->lock); + kfree(migf); + + return 0; +} + +static const struct file_operations qat_vf_save_fops = { + .owner = THIS_MODULE, + .read = qat_vf_save_read, + .unlocked_ioctl = qat_vf_precopy_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .release = qat_vf_release_file, + .llseek = no_llseek, +}; + +static int qat_vf_save_state(struct qat_vf_core_device *qat_vdev, + struct qat_vf_migration_file *migf) +{ + int ret; + + ret = qat_vfmig_save_state(qat_vdev->mdev); + if (ret) + return ret; + migf->filled_size = qat_vdev->mdev->state_size; + + return 0; +} + +static int qat_vf_save_setup(struct qat_vf_core_device *qat_vdev, + struct qat_vf_migration_file *migf) +{ + int ret; + + ret = qat_vfmig_save_setup(qat_vdev->mdev); + if (ret) + return ret; + migf->filled_size = qat_vdev->mdev->setup_size; + + return 0; +} + +/* + * Allocate a file handler for user space and then save the migration data for + * the device being migrated. If this is called in the pre-copy stage, save the + * pre-configured device data. Otherwise, if this is called in the stop-copy + * stage, save the device state. In both cases, update the data size which can + * then be read from user space. + */ +static struct qat_vf_migration_file * +qat_vf_save_device_data(struct qat_vf_core_device *qat_vdev, bool pre_copy) +{ + struct qat_vf_migration_file *migf; + int ret; + + migf = kzalloc(sizeof(*migf), GFP_KERNEL); + if (!migf) + return ERR_PTR(-ENOMEM); + + migf->filp = anon_inode_getfile("qat_vf_mig", &qat_vf_save_fops, + migf, O_RDONLY); + ret = PTR_ERR_OR_ZERO(migf->filp); + if (ret) { + kfree(migf); + return ERR_PTR(ret); + } + + stream_open(migf->filp->f_inode, migf->filp); + mutex_init(&migf->lock); + + if (pre_copy) + ret = qat_vf_save_setup(qat_vdev, migf); + else + ret = qat_vf_save_state(qat_vdev, migf); + if (ret) { + fput(migf->filp); + return ERR_PTR(ret); + } + + migf->qat_vdev = qat_vdev; + + return migf; +} + +static ssize_t qat_vf_resume_write(struct file *filp, const char __user *buf, + size_t len, loff_t *pos) +{ + struct qat_vf_migration_file *migf = filp->private_data; + struct qat_mig_dev *mig_dev = migf->qat_vdev->mdev; + loff_t end, *offs; + ssize_t done = 0; + int ret; + + if (pos) + return -ESPIPE; + offs = &filp->f_pos; + + if (*offs < 0 || + check_add_overflow((loff_t)len, *offs, &end)) + return -EOVERFLOW; + + if (end > mig_dev->state_size) + return -ENOMEM; + + mutex_lock(&migf->lock); + if (migf->disabled) { + done = -ENODEV; + goto out_unlock; + } + + ret = copy_from_user(mig_dev->state + *offs, buf, len); + if (ret) { + done = -EFAULT; + goto out_unlock; + } + *offs += len; + migf->filled_size += len; + + /* + * Load the pre-configured device data first to check if the target + * device is compatible with the source device. + */ + ret = qat_vfmig_load_setup(mig_dev, migf->filled_size); + if (ret && ret != -EAGAIN) { + done = ret; + goto out_unlock; + } + done = len; + +out_unlock: + mutex_unlock(&migf->lock); + return done; +} + +static const struct file_operations qat_vf_resume_fops = { + .owner = THIS_MODULE, + .write = qat_vf_resume_write, + .release = qat_vf_release_file, + .llseek = no_llseek, +}; + +static struct qat_vf_migration_file * +qat_vf_resume_device_data(struct qat_vf_core_device *qat_vdev) +{ + struct qat_vf_migration_file *migf; + int ret; + + migf = kzalloc(sizeof(*migf), GFP_KERNEL); + if (!migf) + return ERR_PTR(-ENOMEM); + + migf->filp = anon_inode_getfile("qat_vf_mig", &qat_vf_resume_fops, migf, O_WRONLY); + ret = PTR_ERR_OR_ZERO(migf->filp); + if (ret) { + kfree(migf); + return ERR_PTR(ret); + } + + migf->qat_vdev = qat_vdev; + migf->filled_size = 0; + stream_open(migf->filp->f_inode, migf->filp); + mutex_init(&migf->lock); + + return migf; +} + +static int qat_vf_load_device_data(struct qat_vf_core_device *qat_vdev) +{ + return qat_vfmig_load_state(qat_vdev->mdev); +} + +static struct file *qat_vf_pci_step_device_state(struct qat_vf_core_device *qat_vdev, u32 new) +{ + u32 cur = qat_vdev->mig_state; + int ret; + + /* + * As the device is not capable of just stopping P2P DMAs, suspend the + * device completely once any of the P2P states are reached. + * When it is suspended, all its MMIO registers can still be operated + * correctly, jobs submitted through ring are queued while no jobs are + * processed by the device. The MMIO states can be safely migrated to + * the target VF during stop-copy stage and restored correctly in the + * target VF. All queued jobs can be resumed then. + */ + if ((cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_RUNNING_P2P) || + (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_PRE_COPY_P2P)) { + ret = qat_vfmig_suspend(qat_vdev->mdev); + if (ret) + return ERR_PTR(ret); + return NULL; + } + + if ((cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_RUNNING) || + (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_PRE_COPY)) { + qat_vfmig_resume(qat_vdev->mdev); + return NULL; + } + + if ((cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_STOP) || + (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING_P2P)) + return NULL; + + if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) { + struct qat_vf_migration_file *migf; + + migf = qat_vf_save_device_data(qat_vdev, false); + if (IS_ERR(migf)) + return ERR_CAST(migf); + get_file(migf->filp); + qat_vdev->saving_migf = migf; + return migf->filp; + } + + if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) { + struct qat_vf_migration_file *migf; + + migf = qat_vf_resume_device_data(qat_vdev); + if (IS_ERR(migf)) + return ERR_CAST(migf); + get_file(migf->filp); + qat_vdev->resuming_migf = migf; + return migf->filp; + } + + if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP) || + (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_RUNNING) || + (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_RUNNING_P2P)) { + qat_vf_disable_fds(qat_vdev); + return NULL; + } + + if ((cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_PRE_COPY) || + (cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_PRE_COPY_P2P)) { + struct qat_vf_migration_file *migf; + + migf = qat_vf_save_device_data(qat_vdev, true); + if (IS_ERR(migf)) + return ERR_CAST(migf); + get_file(migf->filp); + qat_vdev->saving_migf = migf; + return migf->filp; + } + + if (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_STOP_COPY) { + struct qat_vf_migration_file *migf = qat_vdev->saving_migf; + + if (!migf) + return ERR_PTR(-EINVAL); + ret = qat_vf_save_state(qat_vdev, migf); + if (ret) + return ERR_PTR(ret); + return NULL; + } + + if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) { + ret = qat_vf_load_device_data(qat_vdev); + if (ret) + return ERR_PTR(ret); + + qat_vf_disable_fds(qat_vdev); + return NULL; + } + + /* vfio_mig_get_next_state() does not use arcs other than the above */ + WARN_ON(true); + return ERR_PTR(-EINVAL); +} + +static void qat_vf_reset_done(struct qat_vf_core_device *qat_vdev) +{ + qat_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; + qat_vfmig_reset(qat_vdev->mdev); + qat_vf_disable_fds(qat_vdev); +} + +static struct file *qat_vf_pci_set_device_state(struct vfio_device *vdev, + enum vfio_device_mig_state new_state) +{ + struct qat_vf_core_device *qat_vdev = container_of(vdev, + struct qat_vf_core_device, core_device.vdev); + enum vfio_device_mig_state next_state; + struct file *res = NULL; + int ret; + + mutex_lock(&qat_vdev->state_mutex); + while (new_state != qat_vdev->mig_state) { + ret = vfio_mig_get_next_state(vdev, qat_vdev->mig_state, + new_state, &next_state); + if (ret) { + res = ERR_PTR(ret); + break; + } + res = qat_vf_pci_step_device_state(qat_vdev, next_state); + if (IS_ERR(res)) + break; + qat_vdev->mig_state = next_state; + if (WARN_ON(res && new_state != qat_vdev->mig_state)) { + fput(res); + res = ERR_PTR(-EINVAL); + break; + } + } + mutex_unlock(&qat_vdev->state_mutex); + + return res; +} + +static int qat_vf_pci_get_device_state(struct vfio_device *vdev, + enum vfio_device_mig_state *curr_state) +{ + struct qat_vf_core_device *qat_vdev = container_of(vdev, + struct qat_vf_core_device, core_device.vdev); + + mutex_lock(&qat_vdev->state_mutex); + *curr_state = qat_vdev->mig_state; + mutex_unlock(&qat_vdev->state_mutex); + + return 0; +} + +static int qat_vf_pci_get_data_size(struct vfio_device *vdev, + unsigned long *stop_copy_length) +{ + struct qat_vf_core_device *qat_vdev = container_of(vdev, + struct qat_vf_core_device, core_device.vdev); + + mutex_lock(&qat_vdev->state_mutex); + *stop_copy_length = qat_vdev->mdev->state_size; + mutex_unlock(&qat_vdev->state_mutex); + + return 0; +} + +static const struct vfio_migration_ops qat_vf_pci_mig_ops = { + .migration_set_state = qat_vf_pci_set_device_state, + .migration_get_state = qat_vf_pci_get_device_state, + .migration_get_data_size = qat_vf_pci_get_data_size, +}; + +static void qat_vf_pci_release_dev(struct vfio_device *core_vdev) +{ + struct qat_vf_core_device *qat_vdev = container_of(core_vdev, + struct qat_vf_core_device, core_device.vdev); + + qat_vfmig_cleanup(qat_vdev->mdev); + qat_vfmig_destroy(qat_vdev->mdev); + mutex_destroy(&qat_vdev->state_mutex); + vfio_pci_core_release_dev(core_vdev); +} + +static int qat_vf_pci_init_dev(struct vfio_device *core_vdev) +{ + struct qat_vf_core_device *qat_vdev = container_of(core_vdev, + struct qat_vf_core_device, core_device.vdev); + struct qat_mig_dev *mdev; + struct pci_dev *parent; + int ret, vf_id; + + core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P | + VFIO_MIGRATION_PRE_COPY; + core_vdev->mig_ops = &qat_vf_pci_mig_ops; + + ret = vfio_pci_core_init_dev(core_vdev); + if (ret) + return ret; + + mutex_init(&qat_vdev->state_mutex); + + parent = pci_physfn(qat_vdev->core_device.pdev); + vf_id = pci_iov_vf_id(qat_vdev->core_device.pdev); + if (vf_id < 0) { + ret = -ENODEV; + goto err_rel; + } + + mdev = qat_vfmig_create(parent, vf_id); + if (IS_ERR(mdev)) { + ret = PTR_ERR(mdev); + goto err_rel; + } + + ret = qat_vfmig_init(mdev); + if (ret) + goto err_destroy; + + qat_vdev->mdev = mdev; + + return 0; + +err_destroy: + qat_vfmig_destroy(mdev); +err_rel: + vfio_pci_core_release_dev(core_vdev); + return ret; +} + +static const struct vfio_device_ops qat_vf_pci_ops = { + .name = "qat-vf-vfio-pci", + .init = qat_vf_pci_init_dev, + .release = qat_vf_pci_release_dev, + .open_device = qat_vf_pci_open_device, + .close_device = qat_vf_pci_close_device, + .ioctl = vfio_pci_core_ioctl, + .read = vfio_pci_core_read, + .write = vfio_pci_core_write, + .mmap = vfio_pci_core_mmap, + .request = vfio_pci_core_request, + .match = vfio_pci_core_match, + .bind_iommufd = vfio_iommufd_physical_bind, + .unbind_iommufd = vfio_iommufd_physical_unbind, + .attach_ioas = vfio_iommufd_physical_attach_ioas, + .detach_ioas = vfio_iommufd_physical_detach_ioas, +}; + +static struct qat_vf_core_device *qat_vf_drvdata(struct pci_dev *pdev) +{ + struct vfio_pci_core_device *core_device = pci_get_drvdata(pdev); + + return container_of(core_device, struct qat_vf_core_device, core_device); +} + +static void qat_vf_pci_aer_reset_done(struct pci_dev *pdev) +{ + struct qat_vf_core_device *qat_vdev = qat_vf_drvdata(pdev); + + if (!qat_vdev->mdev) + return; + + mutex_lock(&qat_vdev->state_mutex); + qat_vf_reset_done(qat_vdev); + mutex_unlock(&qat_vdev->state_mutex); +} + +static int +qat_vf_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + struct qat_vf_core_device *qat_vdev; + int ret; + + qat_vdev = vfio_alloc_device(qat_vf_core_device, core_device.vdev, dev, &qat_vf_pci_ops); + if (IS_ERR(qat_vdev)) + return PTR_ERR(qat_vdev); + + pci_set_drvdata(pdev, &qat_vdev->core_device); + ret = vfio_pci_core_register_device(&qat_vdev->core_device); + if (ret) + goto out_put_device; + + return 0; + +out_put_device: + vfio_put_device(&qat_vdev->core_device.vdev); + return ret; +} + +static void qat_vf_vfio_pci_remove(struct pci_dev *pdev) +{ + struct qat_vf_core_device *qat_vdev = qat_vf_drvdata(pdev); + + vfio_pci_core_unregister_device(&qat_vdev->core_device); + vfio_put_device(&qat_vdev->core_device.vdev); +} + +static const struct pci_device_id qat_vf_vfio_pci_table[] = { + /* Intel QAT GEN4 4xxx VF device */ + { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, 0x4941) }, + { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, 0x4943) }, + { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, 0x4945) }, + {} +}; +MODULE_DEVICE_TABLE(pci, qat_vf_vfio_pci_table); + +static const struct pci_error_handlers qat_vf_err_handlers = { + .reset_done = qat_vf_pci_aer_reset_done, + .error_detected = vfio_pci_core_aer_err_detected, +}; + +static struct pci_driver qat_vf_vfio_pci_driver = { + .name = "qat_vfio_pci", + .id_table = qat_vf_vfio_pci_table, + .probe = qat_vf_vfio_pci_probe, + .remove = qat_vf_vfio_pci_remove, + .err_handler = &qat_vf_err_handlers, + .driver_managed_dma = true, +}; +module_pci_driver(qat_vf_vfio_pci_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Xin Zeng <xin.zeng@intel.com>"); +MODULE_DESCRIPTION("QAT VFIO PCI - VFIO PCI driver with live migration support for Intel(R) QAT GEN4 device family"); +MODULE_IMPORT_NS(CRYPTO_QAT); diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index d94d61b92c1a..80cae87fff36 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -778,25 +778,26 @@ static int vfio_pci_count_devs(struct pci_dev *pdev, void *data) } struct vfio_pci_fill_info { - struct vfio_pci_dependent_device __user *devices; - struct vfio_pci_dependent_device __user *devices_end; struct vfio_device *vdev; + struct vfio_pci_dependent_device *devices; + int nr_devices; u32 count; u32 flags; }; static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data) { - struct vfio_pci_dependent_device info = { - .segment = pci_domain_nr(pdev->bus), - .bus = pdev->bus->number, - .devfn = pdev->devfn, - }; + struct vfio_pci_dependent_device *info; struct vfio_pci_fill_info *fill = data; - fill->count++; - if (fill->devices >= fill->devices_end) - return 0; + /* The topology changed since we counted devices */ + if (fill->count >= fill->nr_devices) + return -EAGAIN; + + info = &fill->devices[fill->count++]; + info->segment = pci_domain_nr(pdev->bus); + info->bus = pdev->bus->number; + info->devfn = pdev->devfn; if (fill->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID) { struct iommufd_ctx *iommufd = vfio_iommufd_device_ictx(fill->vdev); @@ -809,19 +810,19 @@ static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data) */ vdev = vfio_find_device_in_devset(dev_set, &pdev->dev); if (!vdev) { - info.devid = VFIO_PCI_DEVID_NOT_OWNED; + info->devid = VFIO_PCI_DEVID_NOT_OWNED; } else { int id = vfio_iommufd_get_dev_id(vdev, iommufd); if (id > 0) - info.devid = id; + info->devid = id; else if (id == -ENOENT) - info.devid = VFIO_PCI_DEVID_OWNED; + info->devid = VFIO_PCI_DEVID_OWNED; else - info.devid = VFIO_PCI_DEVID_NOT_OWNED; + info->devid = VFIO_PCI_DEVID_NOT_OWNED; } /* If devid is VFIO_PCI_DEVID_NOT_OWNED, clear owned flag. */ - if (info.devid == VFIO_PCI_DEVID_NOT_OWNED) + if (info->devid == VFIO_PCI_DEVID_NOT_OWNED) fill->flags &= ~VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED; } else { struct iommu_group *iommu_group; @@ -830,13 +831,10 @@ static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data) if (!iommu_group) return -EPERM; /* Cannot reset non-isolated devices */ - info.group_id = iommu_group_id(iommu_group); + info->group_id = iommu_group_id(iommu_group); iommu_group_put(iommu_group); } - if (copy_to_user(fill->devices, &info, sizeof(info))) - return -EFAULT; - fill->devices++; return 0; } @@ -1258,10 +1256,11 @@ static int vfio_pci_ioctl_get_pci_hot_reset_info( { unsigned long minsz = offsetofend(struct vfio_pci_hot_reset_info, count); + struct vfio_pci_dependent_device *devices = NULL; struct vfio_pci_hot_reset_info hdr; struct vfio_pci_fill_info fill = {}; bool slot = false; - int ret = 0; + int ret, count; if (copy_from_user(&hdr, arg, minsz)) return -EFAULT; @@ -1277,9 +1276,26 @@ static int vfio_pci_ioctl_get_pci_hot_reset_info( else if (pci_probe_reset_bus(vdev->pdev->bus)) return -ENODEV; - fill.devices = arg->devices; - fill.devices_end = arg->devices + - (hdr.argsz - sizeof(hdr)) / sizeof(arg->devices[0]); + ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs, + &count, slot); + if (ret) + return ret; + + if (WARN_ON(!count)) /* Should always be at least one */ + return -ERANGE; + + if (count > (hdr.argsz - sizeof(hdr)) / sizeof(*devices)) { + hdr.count = count; + ret = -ENOSPC; + goto header; + } + + devices = kcalloc(count, sizeof(*devices), GFP_KERNEL); + if (!devices) + return -ENOMEM; + + fill.devices = devices; + fill.nr_devices = count; fill.vdev = &vdev->vdev; if (vfio_device_cdev_opened(&vdev->vdev)) @@ -1291,16 +1307,23 @@ static int vfio_pci_ioctl_get_pci_hot_reset_info( &fill, slot); mutex_unlock(&vdev->vdev.dev_set->lock); if (ret) - return ret; + goto out; + + if (copy_to_user(arg->devices, devices, + sizeof(*devices) * fill.count)) { + ret = -EFAULT; + goto out; + } hdr.count = fill.count; hdr.flags = fill.flags; - if (copy_to_user(arg, &hdr, minsz)) - return -EFAULT; - if (fill.count > fill.devices - arg->devices) - return -ENOSPC; - return 0; +header: + if (copy_to_user(arg, &hdr, minsz)) + ret = -EFAULT; +out: + kfree(devices); + return ret; } static int diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index fb5392b749ff..8382c5834335 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -23,11 +23,12 @@ #include "vfio_pci_priv.h" struct vfio_pci_irq_ctx { - struct eventfd_ctx *trigger; - struct virqfd *unmask; - struct virqfd *mask; - char *name; - bool masked; + struct vfio_pci_core_device *vdev; + struct eventfd_ctx *trigger; + struct virqfd *unmask; + struct virqfd *mask; + char *name; + bool masked; struct irq_bypass_producer producer; }; @@ -84,19 +85,14 @@ vfio_irq_ctx_alloc(struct vfio_pci_core_device *vdev, unsigned long index) /* * INTx */ -static void vfio_send_intx_eventfd(void *opaque, void *unused) +static void vfio_send_intx_eventfd(void *opaque, void *data) { struct vfio_pci_core_device *vdev = opaque; if (likely(is_intx(vdev) && !vdev->virq_disabled)) { - struct vfio_pci_irq_ctx *ctx; - struct eventfd_ctx *trigger; + struct vfio_pci_irq_ctx *ctx = data; + struct eventfd_ctx *trigger = READ_ONCE(ctx->trigger); - ctx = vfio_irq_ctx_get(vdev, 0); - if (WARN_ON_ONCE(!ctx)) - return; - - trigger = READ_ONCE(ctx->trigger); if (likely(trigger)) eventfd_signal(trigger); } @@ -166,11 +162,11 @@ bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) * a signal is necessary, which can then be handled via a work queue * or directly depending on the caller. */ -static int vfio_pci_intx_unmask_handler(void *opaque, void *unused) +static int vfio_pci_intx_unmask_handler(void *opaque, void *data) { struct vfio_pci_core_device *vdev = opaque; struct pci_dev *pdev = vdev->pdev; - struct vfio_pci_irq_ctx *ctx; + struct vfio_pci_irq_ctx *ctx = data; unsigned long flags; int ret = 0; @@ -186,10 +182,6 @@ static int vfio_pci_intx_unmask_handler(void *opaque, void *unused) goto out_unlock; } - ctx = vfio_irq_ctx_get(vdev, 0); - if (WARN_ON_ONCE(!ctx)) - goto out_unlock; - if (ctx->masked && !vdev->virq_disabled) { /* * A pending interrupt here would immediately trigger, @@ -213,10 +205,12 @@ out_unlock: static void __vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) { + struct vfio_pci_irq_ctx *ctx = vfio_irq_ctx_get(vdev, 0); + lockdep_assert_held(&vdev->igate); - if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0) - vfio_send_intx_eventfd(vdev, NULL); + if (vfio_pci_intx_unmask_handler(vdev, ctx) > 0) + vfio_send_intx_eventfd(vdev, ctx); } void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) @@ -228,15 +222,11 @@ void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) static irqreturn_t vfio_intx_handler(int irq, void *dev_id) { - struct vfio_pci_core_device *vdev = dev_id; - struct vfio_pci_irq_ctx *ctx; + struct vfio_pci_irq_ctx *ctx = dev_id; + struct vfio_pci_core_device *vdev = ctx->vdev; unsigned long flags; int ret = IRQ_NONE; - ctx = vfio_irq_ctx_get(vdev, 0); - if (WARN_ON_ONCE(!ctx)) - return ret; - spin_lock_irqsave(&vdev->irqlock, flags); if (!vdev->pci_2_3) { @@ -252,7 +242,7 @@ static irqreturn_t vfio_intx_handler(int irq, void *dev_id) spin_unlock_irqrestore(&vdev->irqlock, flags); if (ret == IRQ_HANDLED) - vfio_send_intx_eventfd(vdev, NULL); + vfio_send_intx_eventfd(vdev, ctx); return ret; } @@ -277,11 +267,14 @@ static int vfio_intx_enable(struct vfio_pci_core_device *vdev, return -ENOMEM; ctx = vfio_irq_ctx_alloc(vdev, 0); - if (!ctx) + if (!ctx) { + kfree(name); return -ENOMEM; + } ctx->name = name; ctx->trigger = trigger; + ctx->vdev = vdev; /* * Fill the initial masked state based on virq_disabled. After @@ -312,7 +305,7 @@ static int vfio_intx_enable(struct vfio_pci_core_device *vdev, vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX; ret = request_irq(pdev->irq, vfio_intx_handler, - irqflags, ctx->name, vdev); + irqflags, ctx->name, ctx); if (ret) { vdev->irq_type = VFIO_PCI_NUM_IRQS; kfree(name); @@ -358,7 +351,7 @@ static void vfio_intx_disable(struct vfio_pci_core_device *vdev) if (ctx) { vfio_virqfd_disable(&ctx->unmask); vfio_virqfd_disable(&ctx->mask); - free_irq(pdev->irq, vdev); + free_irq(pdev->irq, ctx); if (ctx->trigger) eventfd_ctx_put(ctx->trigger); kfree(ctx->name); @@ -606,7 +599,7 @@ static int vfio_pci_set_intx_unmask(struct vfio_pci_core_device *vdev, if (fd >= 0) return vfio_virqfd_enable((void *) vdev, vfio_pci_intx_unmask_handler, - vfio_send_intx_eventfd, NULL, + vfio_send_intx_eventfd, ctx, &ctx->unmask, fd); vfio_virqfd_disable(&ctx->unmask); @@ -673,11 +666,11 @@ static int vfio_pci_set_intx_trigger(struct vfio_pci_core_device *vdev, return -EINVAL; if (flags & VFIO_IRQ_SET_DATA_NONE) { - vfio_send_intx_eventfd(vdev, NULL); + vfio_send_intx_eventfd(vdev, vfio_irq_ctx_get(vdev, 0)); } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { uint8_t trigger = *(uint8_t *)data; if (trigger) - vfio_send_intx_eventfd(vdev, NULL); + vfio_send_intx_eventfd(vdev, vfio_irq_ctx_get(vdev, 0)); } return 0; } diff --git a/drivers/vfio/platform/vfio_amba.c b/drivers/vfio/platform/vfio_amba.c index 485c6f9161a9..ff8ff8480968 100644 --- a/drivers/vfio/platform/vfio_amba.c +++ b/drivers/vfio/platform/vfio_amba.c @@ -134,7 +134,6 @@ static struct amba_driver vfio_amba_driver = { .id_table = vfio_amba_ids, .drv = { .name = "vfio-amba", - .owner = THIS_MODULE, }, .driver_managed_dma = true, }; diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index b5c15fe8f9fc..3a0218171cfa 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -518,7 +518,7 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, spinlock_t *ptl; int ret; - ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); + ret = follow_pte(vma, vaddr, &ptep, &ptl); if (ret) { bool unlocked = false; @@ -532,7 +532,7 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, if (ret) return ret; - ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); + ret = follow_pte(vma, vaddr, &ptep, &ptl); if (ret) return ret; } diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index fcabc668e9fb..3f7333dca508 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -2907,7 +2907,7 @@ void fbcon_remap_all(struct fb_info *info) static void fbcon_select_primary(struct fb_info *info) { if (!map_override && primary_device == -1 && - fb_is_primary_device(info)) { + video_is_primary_device(info->device)) { int i; printk(KERN_INFO "fbcon: %s (fb%i) is primary device\n", diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig index 40129b6f0eca..d8c848cf09a6 100644 --- a/drivers/virt/Kconfig +++ b/drivers/virt/Kconfig @@ -16,7 +16,6 @@ if VIRT_DRIVERS config VMGENID tristate "Virtual Machine Generation ID driver" default y - depends on ACPI help Say Y here to use the hypervisor-provided Virtual Machine Generation ID to reseed the RNG when the VM is cloned. This is highly recommended if diff --git a/drivers/virt/acrn/mm.c b/drivers/virt/acrn/mm.c index fa5d9ca6be57..db8ff1d0ac23 100644 --- a/drivers/virt/acrn/mm.c +++ b/drivers/virt/acrn/mm.c @@ -12,6 +12,7 @@ #include <linux/io.h> #include <linux/mm.h> #include <linux/slab.h> +#include <linux/vmalloc.h> #include "acrn_drv.h" @@ -155,43 +156,82 @@ int acrn_vm_memseg_unmap(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) { struct vm_memory_region_batch *regions_info; - int nr_pages, i = 0, order, nr_regions = 0; + int nr_pages, i, order, nr_regions = 0; struct vm_memory_mapping *region_mapping; struct vm_memory_region_op *vm_region; struct page **pages = NULL, *page; void *remap_vaddr; int ret, pinned; u64 user_vm_pa; - unsigned long pfn; struct vm_area_struct *vma; if (!vm || !memmap) return -EINVAL; + /* Get the page number of the map region */ + nr_pages = memmap->len >> PAGE_SHIFT; + if (!nr_pages) + return -EINVAL; + mmap_read_lock(current->mm); vma = vma_lookup(current->mm, memmap->vma_base); if (vma && ((vma->vm_flags & VM_PFNMAP) != 0)) { + unsigned long start_pfn, cur_pfn; + spinlock_t *ptl; + bool writable; + pte_t *ptep; + if ((memmap->vma_base + memmap->len) > vma->vm_end) { mmap_read_unlock(current->mm); return -EINVAL; } - ret = follow_pfn(vma, memmap->vma_base, &pfn); + for (i = 0; i < nr_pages; i++) { + ret = follow_pte(vma, memmap->vma_base + i * PAGE_SIZE, + &ptep, &ptl); + if (ret) + break; + + cur_pfn = pte_pfn(ptep_get(ptep)); + if (i == 0) + start_pfn = cur_pfn; + writable = !!pte_write(ptep_get(ptep)); + pte_unmap_unlock(ptep, ptl); + + /* Disallow write access if the PTE is not writable. */ + if (!writable && + (memmap->attr & ACRN_MEM_ACCESS_WRITE)) { + ret = -EFAULT; + break; + } + + /* Disallow refcounted pages. */ + if (pfn_valid(cur_pfn) && + !PageReserved(pfn_to_page(cur_pfn))) { + ret = -EFAULT; + break; + } + + /* Disallow non-contiguous ranges. */ + if (cur_pfn != start_pfn + i) { + ret = -EINVAL; + break; + } + } mmap_read_unlock(current->mm); - if (ret < 0) { + + if (ret) { dev_dbg(acrn_dev.this_device, "Failed to lookup PFN at VMA:%pK.\n", (void *)memmap->vma_base); return ret; } return acrn_mm_region_add(vm, memmap->user_vm_pa, - PFN_PHYS(pfn), memmap->len, + PFN_PHYS(start_pfn), memmap->len, ACRN_MEM_TYPE_WB, memmap->attr); } mmap_read_unlock(current->mm); - /* Get the page number of the map region */ - nr_pages = memmap->len >> PAGE_SHIFT; pages = vzalloc(array_size(nr_pages, sizeof(*pages))); if (!pages) return -ENOMEM; @@ -235,12 +275,11 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) mutex_unlock(&vm->regions_mapping_lock); /* Calculate count of vm_memory_region_op */ - while (i < nr_pages) { + for (i = 0; i < nr_pages; i += 1 << order) { page = pages[i]; VM_BUG_ON_PAGE(PageTail(page), page); order = compound_order(page); nr_regions++; - i += 1 << order; } /* Prepare the vm_memory_region_batch */ @@ -257,8 +296,7 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) regions_info->vmid = vm->vmid; regions_info->regions_gpa = virt_to_phys(vm_region); user_vm_pa = memmap->user_vm_pa; - i = 0; - while (i < nr_pages) { + for (i = 0; i < nr_pages; i += 1 << order) { u32 region_size; page = pages[i]; @@ -274,7 +312,6 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) vm_region++; user_vm_pa += region_size; - i += 1 << order; } /* Inform the ACRN Hypervisor to set up EPT mappings */ diff --git a/drivers/virt/vmgenid.c b/drivers/virt/vmgenid.c index eba4250ab3cf..66135eac3abf 100644 --- a/drivers/virt/vmgenid.c +++ b/drivers/virt/vmgenid.c @@ -2,14 +2,16 @@ /* * Copyright (C) 2022 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. * - * The "Virtual Machine Generation ID" is exposed via ACPI and changes when a + * The "Virtual Machine Generation ID" is exposed via ACPI or DT and changes when a * virtual machine forks or is cloned. This driver exists for shepherding that * information to random.c. */ +#include <linux/acpi.h> +#include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/module.h> -#include <linux/acpi.h> +#include <linux/platform_device.h> #include <linux/random.h> ACPI_MODULE_NAME("vmgenid"); @@ -21,19 +23,42 @@ struct vmgenid_state { u8 this_id[VMGENID_SIZE]; }; -static int vmgenid_add(struct acpi_device *device) +static void vmgenid_notify(struct device *device) +{ + struct vmgenid_state *state = device->driver_data; + u8 old_id[VMGENID_SIZE]; + + memcpy(old_id, state->this_id, sizeof(old_id)); + memcpy(state->this_id, state->next_id, sizeof(state->this_id)); + if (!memcmp(old_id, state->this_id, sizeof(old_id))) + return; + add_vmfork_randomness(state->this_id, sizeof(state->this_id)); +} + +static void setup_vmgenid_state(struct vmgenid_state *state, void *virt_addr) +{ + state->next_id = virt_addr; + memcpy(state->this_id, state->next_id, sizeof(state->this_id)); + add_device_randomness(state->this_id, sizeof(state->this_id)); +} + +#ifdef CONFIG_ACPI +static void vmgenid_acpi_handler(acpi_handle __always_unused handle, + u32 __always_unused event, void *dev) +{ + vmgenid_notify(dev); +} + +static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state) { + struct acpi_device *device = ACPI_COMPANION(dev); struct acpi_buffer parsed = { ACPI_ALLOCATE_BUFFER }; - struct vmgenid_state *state; union acpi_object *obj; phys_addr_t phys_addr; acpi_status status; + void *virt_addr; int ret = 0; - state = devm_kmalloc(&device->dev, sizeof(*state), GFP_KERNEL); - if (!state) - return -ENOMEM; - status = acpi_evaluate_object(device->handle, "ADDR", NULL, &parsed); if (ACPI_FAILURE(status)) { ACPI_EXCEPTION((AE_INFO, status, "Evaluating ADDR")); @@ -49,52 +74,108 @@ static int vmgenid_add(struct acpi_device *device) phys_addr = (obj->package.elements[0].integer.value << 0) | (obj->package.elements[1].integer.value << 32); - state->next_id = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE, MEMREMAP_WB); - if (IS_ERR(state->next_id)) { - ret = PTR_ERR(state->next_id); + + virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE, MEMREMAP_WB); + if (IS_ERR(virt_addr)) { + ret = PTR_ERR(virt_addr); goto out; } + setup_vmgenid_state(state, virt_addr); - memcpy(state->this_id, state->next_id, sizeof(state->this_id)); - add_device_randomness(state->this_id, sizeof(state->this_id)); - - device->driver_data = state; + status = acpi_install_notify_handler(device->handle, ACPI_DEVICE_NOTIFY, + vmgenid_acpi_handler, dev); + if (ACPI_FAILURE(status)) { + ret = -ENODEV; + goto out; + } + dev->driver_data = state; out: ACPI_FREE(parsed.pointer); return ret; } +#else +static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state) +{ + return -EINVAL; +} +#endif -static void vmgenid_notify(struct acpi_device *device, u32 event) +static irqreturn_t vmgenid_of_irq_handler(int __always_unused irq, void *dev) { - struct vmgenid_state *state = acpi_driver_data(device); - u8 old_id[VMGENID_SIZE]; + vmgenid_notify(dev); + return IRQ_HANDLED; +} - memcpy(old_id, state->this_id, sizeof(old_id)); - memcpy(state->this_id, state->next_id, sizeof(state->this_id)); - if (!memcmp(old_id, state->this_id, sizeof(old_id))) - return; - add_vmfork_randomness(state->this_id, sizeof(state->this_id)); +static int vmgenid_add_of(struct platform_device *pdev, + struct vmgenid_state *state) +{ + void *virt_addr; + int ret; + + virt_addr = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(virt_addr)) + return PTR_ERR(virt_addr); + + setup_vmgenid_state(state, virt_addr); + + ret = platform_get_irq(pdev, 0); + if (ret < 0) + return ret; + + ret = devm_request_irq(&pdev->dev, ret, vmgenid_of_irq_handler, + IRQF_SHARED, "vmgenid", &pdev->dev); + if (ret < 0) + return ret; + + pdev->dev.driver_data = state; + return 0; } -static const struct acpi_device_id vmgenid_ids[] = { +static int vmgenid_add(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct vmgenid_state *state; + int ret; + + state = devm_kmalloc(dev, sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + if (dev->of_node) + ret = vmgenid_add_of(pdev, state); + else + ret = vmgenid_add_acpi(dev, state); + + if (ret < 0) + devm_kfree(dev, state); + return ret; +} + +static const struct of_device_id vmgenid_of_ids[] = { + { .compatible = "microsoft,vmgenid", }, + { }, +}; +MODULE_DEVICE_TABLE(of, vmgenid_of_ids); + +static const struct acpi_device_id vmgenid_acpi_ids[] = { { "VMGENCTR", 0 }, { "VM_GEN_COUNTER", 0 }, { } }; - -static struct acpi_driver vmgenid_driver = { - .name = "vmgenid", - .ids = vmgenid_ids, - .ops = { - .add = vmgenid_add, - .notify = vmgenid_notify - } +MODULE_DEVICE_TABLE(acpi, vmgenid_acpi_ids); + +static struct platform_driver vmgenid_plaform_driver = { + .probe = vmgenid_add, + .driver = { + .name = "vmgenid", + .acpi_match_table = vmgenid_acpi_ids, + .of_match_table = vmgenid_of_ids, + }, }; -module_acpi_driver(vmgenid_driver); +module_platform_driver(vmgenid_plaform_driver) -MODULE_DEVICE_TABLE(acpi, vmgenid_ids); MODULE_DESCRIPTION("Virtual Machine Generation ID"); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 8e3223294442..e8355f55a8f7 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -21,6 +21,7 @@ #include <linux/bitmap.h> #include <linux/lockdep.h> #include <linux/log2.h> +#include <linux/vmalloc.h> #include <acpi/acpi_numa.h> diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index 6bee137cfbe0..85eea38dbdf4 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -254,6 +254,17 @@ config GPIO_WATCHDOG_ARCH_INITCALL arch_initcall. If in doubt, say N. +config LENOVO_SE10_WDT + tristate "Lenovo SE10 Watchdog" + depends on (X86 && DMI) || COMPILE_TEST + select WATCHDOG_CORE + help + If you say yes here you get support for the watchdog + functionality for the Lenovo SE10 platform. + + This driver can also be built as a module. If so, the module + will be called lenovo-se10-wdt. + config MENF21BMC_WATCHDOG tristate "MEN 14F021P00 BMC Watchdog" depends on MFD_MENF21BMC || COMPILE_TEST @@ -482,6 +493,7 @@ config 21285_WATCHDOG config 977_WATCHDOG tristate "NetWinder WB83C977 watchdog" depends on (FOOTBRIDGE && ARCH_NETWINDER) || (ARM && COMPILE_TEST) + depends on HAS_IOPORT help Say Y here to include support for the WB977 watchdog included in NetWinder machines. Alternatively say M to compile the driver as @@ -1075,7 +1087,7 @@ config ACQUIRE_WDT config ADVANTECH_WDT tristate "Advantech SBC Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help If you are configuring a Linux kernel for the Advantech single-board computer, say `Y' here to support its built-in watchdog timer @@ -1084,7 +1096,7 @@ config ADVANTECH_WDT config ADVANTECH_EC_WDT tristate "Advantech Embedded Controller Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select ISA_BUS_API select WATCHDOG_CORE help @@ -1117,7 +1129,7 @@ config ALIM7101_WDT config EBC_C384_WDT tristate "WinSystems EBC-C384 Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select ISA_BUS_API select WATCHDOG_CORE help @@ -1127,7 +1139,7 @@ config EBC_C384_WDT config EXAR_WDT tristate "Exar Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select WATCHDOG_CORE help Enables watchdog timer support for the watchdog timer present @@ -1138,7 +1150,7 @@ config EXAR_WDT config F71808E_WDT tristate "Fintek F718xx, F818xx Super I/O Watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select WATCHDOG_CORE help This is the driver for the hardware watchdog on the Fintek F71808E, @@ -1150,7 +1162,7 @@ config F71808E_WDT config SP5100_TCO tristate "AMD/ATI SP5100 TCO Timer/Watchdog" - depends on (X86 || COMPILE_TEST) && PCI + depends on (X86 || COMPILE_TEST) && PCI && HAS_IOPORT select WATCHDOG_CORE help Hardware watchdog driver for the AMD/ATI SP5100 chipset. The TCO @@ -1189,7 +1201,7 @@ config SC520_WDT config SBC_FITPC2_WATCHDOG tristate "Compulab SBC-FITPC2 watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the built-in watchdog timer on the fit-PC2, fit-PC2i, CM-iAM single-board computers made by Compulab. @@ -1214,7 +1226,7 @@ config SBC_FITPC2_WATCHDOG config EUROTECH_WDT tristate "Eurotech CPU-1220/1410 Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help Enable support for the watchdog timer on the Eurotech CPU-1220 and CPU-1410 cards. These are PC/104 SBCs. Spec sheets and product @@ -1222,7 +1234,7 @@ config EUROTECH_WDT config IB700_WDT tristate "IB700 SBC Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog on the IB700 Single Board Computer produced by TMC Technology (www.tmc-uk.com). This @@ -1239,7 +1251,7 @@ config IB700_WDT config IBMASR tristate "IBM Automatic Server Restart" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the IBM Automatic Server Restart watchdog timer built-in into some eServer xSeries machines. @@ -1249,7 +1261,7 @@ config IBMASR config WAFER_WDT tristate "ICP Single Board Computer Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is a driver for the hardware watchdog on the ICP Single Board Computer. This driver is working on (at least) the following @@ -1271,7 +1283,7 @@ config I6300ESB_WDT config IE6XX_WDT tristate "Intel Atom E6xx Watchdog" - depends on (X86 || COMPILE_TEST) && PCI + depends on (X86 || COMPILE_TEST) && PCI && HAS_IOPORT select WATCHDOG_CORE select MFD_CORE select LPC_SCH @@ -1301,6 +1313,7 @@ config ITCO_WDT select WATCHDOG_CORE depends on I2C || I2C=n depends on MFD_INTEL_PMC_BXT || !MFD_INTEL_PMC_BXT + depends on HAS_IOPORT # for I2C_I801 select LPC_ICH if !EXPERT select I2C_I801 if !EXPERT && I2C help @@ -1331,7 +1344,7 @@ config ITCO_VENDOR_SUPPORT config IT8712F_WDT tristate "IT8712F (Smart Guardian) Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the built-in watchdog timer on the IT8712F Super I/0 chipset used on many motherboards. @@ -1344,7 +1357,7 @@ config IT8712F_WDT config IT87_WDT tristate "IT87 Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select WATCHDOG_CORE help This is the driver for the hardware watchdog on the ITE IT8607, @@ -1392,7 +1405,7 @@ config KEMPLD_WDT config SC1200_WDT tristate "National Semiconductor PC87307/PC97307 (ala SC1200) Watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is a driver for National Semiconductor PC87307/PC97307 hardware watchdog cards as found on the SC1200. This watchdog is mainly used @@ -1415,7 +1428,7 @@ config SCx200_WDT config PC87413_WDT tristate "NS PC87413 watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog on the PC87413 chipset This watchdog simply watches your kernel to make sure it doesn't @@ -1429,7 +1442,7 @@ config PC87413_WDT config NV_TCO tristate "nVidia TCO Timer/Watchdog" - depends on (X86 || COMPILE_TEST) && PCI + depends on (X86 || COMPILE_TEST) && PCI && HAS_IOPORT help Hardware driver for the TCO timer built into the nVidia Hub family (such as the MCP51). The TCO (Total Cost of Ownership) timer is a @@ -1458,7 +1471,7 @@ config RDC321X_WDT config 60XX_WDT tristate "SBC-60XX Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This driver can be used with the watchdog timer found on some single board computers, namely the 6010 PII based computer. @@ -1498,7 +1511,7 @@ config SBC7240_WDT config CPU5_WDT tristate "SMA CPU5 Watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help TBD. To compile this driver as a module, choose M here: the @@ -1506,7 +1519,7 @@ config CPU5_WDT config SMSC_SCH311X_WDT tristate "SMSC SCH311X Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog timer on the SMSC SCH3112, SCH3114 and SCH3116 Super IO chipset @@ -1518,7 +1531,7 @@ config SMSC_SCH311X_WDT config SMSC37B787_WDT tristate "Winbond SMsC37B787 Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog component on the Winbond SMsC37B787 chipset as used on the NetRunner Mainboard @@ -1564,7 +1577,7 @@ config VIA_WDT config W83627HF_WDT tristate "Watchdog timer for W83627HF/W83627DHG and compatibles" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select WATCHDOG_CORE help This is the driver for the hardware watchdog on the following @@ -1594,7 +1607,7 @@ config W83627HF_WDT config W83877F_WDT tristate "W83877F (EMACS) Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog on the W83877F chipset as used in EMACS PC-104 motherboards (and likely others). This @@ -1609,7 +1622,7 @@ config W83877F_WDT config W83977F_WDT tristate "W83977F (PCM-5335) Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog on the W83977F I/O chip as used in AAEON's PCM-5335 SBC (and likely others). This @@ -1622,7 +1635,7 @@ config W83977F_WDT config MACHZ_WDT tristate "ZF MachZ Watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help If you are using a ZF Micro MachZ processor, say Y here, otherwise N. This is the driver for the watchdog timer built-in on that @@ -1635,7 +1648,7 @@ config MACHZ_WDT config SBC_EPX_C3_WATCHDOG tristate "Winsystems SBC EPX-C3 watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the built-in watchdog timer on the EPX-C3 Single-board computer made by Winsystems, Inc. @@ -2197,7 +2210,7 @@ comment "PCI-based Watchdog Cards" config PCIPCWATCHDOG tristate "Berkshire Products PCI-PC Watchdog" - depends on PCI + depends on PCI && HAS_IOPORT help This is the driver for the Berkshire Products PCI-PC Watchdog card. This card simply watches your kernel to make sure it doesn't freeze, @@ -2212,7 +2225,7 @@ config PCIPCWATCHDOG config WDTPCI tristate "PCI-WDT500/501 Watchdog timer" - depends on PCI + depends on PCI && HAS_IOPORT help If you have a PCI-WDT500/501 watchdog board, say Y here, otherwise N. diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile index 3710c218f05e..2d1117564f5b 100644 --- a/drivers/watchdog/Makefile +++ b/drivers/watchdog/Makefile @@ -120,6 +120,7 @@ obj-$(CONFIG_WAFER_WDT) += wafer5823wdt.o obj-$(CONFIG_I6300ESB_WDT) += i6300esb.o obj-$(CONFIG_IE6XX_WDT) += ie6xx_wdt.o obj-$(CONFIG_ITCO_WDT) += iTCO_wdt.o +obj-$(CONFIG_LENOVO_SE10_WDT) += lenovo_se10_wdt.o ifeq ($(CONFIG_ITCO_VENDOR_SUPPORT),y) obj-$(CONFIG_ITCO_WDT) += iTCO_vendor_support.o endif diff --git a/drivers/watchdog/bd9576_wdt.c b/drivers/watchdog/bd9576_wdt.c index 4a20e07fbb69..f00ea1b4e40b 100644 --- a/drivers/watchdog/bd9576_wdt.c +++ b/drivers/watchdog/bd9576_wdt.c @@ -29,7 +29,6 @@ struct bd9576_wdt_priv { struct gpio_desc *gpiod_en; struct device *dev; struct regmap *regmap; - bool always_running; struct watchdog_device wdd; }; @@ -62,10 +61,7 @@ static int bd9576_wdt_stop(struct watchdog_device *wdd) { struct bd9576_wdt_priv *priv = watchdog_get_drvdata(wdd); - if (!priv->always_running) - bd9576_wdt_disable(priv); - else - set_bit(WDOG_HW_RUNNING, &wdd->status); + bd9576_wdt_disable(priv); return 0; } @@ -264,9 +260,6 @@ static int bd9576_wdt_probe(struct platform_device *pdev) if (ret) return ret; - priv->always_running = device_property_read_bool(dev->parent, - "always-running"); - watchdog_set_drvdata(&priv->wdd, priv); priv->wdd.info = &bd957x_wdt_ident; @@ -281,9 +274,6 @@ static int bd9576_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(&priv->wdd); - if (priv->always_running) - bd9576_wdt_start(&priv->wdd); - return devm_watchdog_register_device(dev, &priv->wdd); } diff --git a/drivers/watchdog/cpu5wdt.c b/drivers/watchdog/cpu5wdt.c index 688b112e712b..9f279c0e13a6 100644 --- a/drivers/watchdog/cpu5wdt.c +++ b/drivers/watchdog/cpu5wdt.c @@ -252,7 +252,7 @@ static void cpu5wdt_exit(void) if (cpu5wdt_device.queue) { cpu5wdt_device.queue = 0; wait_for_completion(&cpu5wdt_device.stop); - del_timer(&cpu5wdt_device.timer); + timer_shutdown_sync(&cpu5wdt_device.timer); } misc_deregister(&cpu5wdt_misc); diff --git a/drivers/watchdog/lenovo_se10_wdt.c b/drivers/watchdog/lenovo_se10_wdt.c new file mode 100644 index 000000000000..139ff0e8220f --- /dev/null +++ b/drivers/watchdog/lenovo_se10_wdt.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * WDT driver for Lenovo SE10. + */ + +#include <linux/delay.h> +#include <linux/dmi.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/platform_device.h> +#include <linux/string.h> +#include <linux/types.h> +#include <linux/watchdog.h> + +#define STATUS_PORT 0x6C +#define CMD_PORT 0x6C +#define DATA_PORT 0x68 +#define OUTBUF_FULL 0x01 +#define INBUF_EMPTY 0x02 +#define CFG_LDN 0x07 +#define CFG_BRAM_LDN 0x10 /* for BRAM Base */ +#define CFG_PORT 0x2E +#define CFG_SIZE 2 +#define CMD_SIZE 4 +#define BRAM_SIZE 2 + +#define UNLOCK_KEY 0x87 +#define LOCK_KEY 0xAA + +#define CUS_WDT_SWI 0x1A +#define CUS_WDT_CFG 0x1B +#define CUS_WDT_FEED 0xB0 +#define CUS_WDT_CNT 0xB1 + +#define DRVNAME "lenovo-se10-wdt" + +/*The timeout range is 1-255 seconds*/ +#define MIN_TIMEOUT 1 +#define MAX_TIMEOUT 255 +#define MAX_WAIT 10 + +#define WATCHDOG_TIMEOUT 60 /* 60 sec default timeout */ +static unsigned short bram_base; +static struct platform_device *se10_pdev; + +static int timeout; /* in seconds */ +module_param(timeout, int, 0); +MODULE_PARM_DESC(timeout, + "Watchdog timeout in seconds. 1 <= timeout <= 255, default=" + __MODULE_STRING(WATCHDOG_TIMEOUT) "."); + +static bool nowayout = WATCHDOG_NOWAYOUT; +module_param(nowayout, bool, 0); +MODULE_PARM_DESC(nowayout, + "Watchdog cannot be stopped once started (default=" + __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); + +struct se10_wdt { + struct watchdog_device wdd; +}; + +static int set_bram(unsigned char offset, unsigned char val) +{ + if (!request_muxed_region(bram_base, BRAM_SIZE, DRVNAME)) + return -EBUSY; + outb(offset, bram_base); + outb(val, bram_base + 1); + release_region(bram_base, BRAM_SIZE); + return 0; +} + +static void wait_for_buffer(int condition) +{ + int loop = 0; + + while (1) { + if (inb(STATUS_PORT) & condition || loop > MAX_WAIT) + break; + loop++; + usleep_range(10, 125); + } +} + +static void send_cmd(unsigned char cmd) +{ + wait_for_buffer(INBUF_EMPTY); + outb(cmd, CMD_PORT); + wait_for_buffer(INBUF_EMPTY); +} + +static void lpc_write(unsigned char index, unsigned char data) +{ + outb(index, CFG_PORT); + outb(data, CFG_PORT + 1); +} + +static unsigned char lpc_read(unsigned char index) +{ + outb(index, CFG_PORT); + return inb(CFG_PORT + 1); +} + +static int wdt_start(struct watchdog_device *wdog) +{ + return set_bram(CUS_WDT_SWI, 0x80); +} + +static int wdt_set_timeout(struct watchdog_device *wdog, unsigned int timeout) +{ + wdog->timeout = timeout; + return set_bram(CUS_WDT_CFG, wdog->timeout); +} + +static int wdt_stop(struct watchdog_device *wdog) +{ + return set_bram(CUS_WDT_SWI, 0); +} + +static unsigned int wdt_get_time(struct watchdog_device *wdog) +{ + unsigned char time; + + if (!request_muxed_region(CMD_PORT, CMD_SIZE, DRVNAME)) + return -EBUSY; + send_cmd(CUS_WDT_CNT); + wait_for_buffer(OUTBUF_FULL); + time = inb(DATA_PORT); + release_region(CMD_PORT, CMD_SIZE); + return time; +} + +static int wdt_ping(struct watchdog_device *wdog) +{ + if (!request_muxed_region(CMD_PORT, CMD_SIZE, DRVNAME)) + return -EBUSY; + send_cmd(CUS_WDT_FEED); + release_region(CMD_PORT, CMD_SIZE); + return 0; +} + +static const struct watchdog_info wdt_info = { + .options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE, + .identity = "Lenovo SE10 Watchdog", +}; + +static const struct watchdog_ops se10_wdt_ops = { + .owner = THIS_MODULE, + .start = wdt_start, + .stop = wdt_stop, + .ping = wdt_ping, + .set_timeout = wdt_set_timeout, + .get_timeleft = wdt_get_time, +}; + +static unsigned int get_chipID(void) +{ + unsigned char msb, lsb; + + outb(UNLOCK_KEY, CFG_PORT); + outb(0x01, CFG_PORT); + outb(0x55, CFG_PORT); + outb(0x55, CFG_PORT); + msb = lpc_read(0x20); + lsb = lpc_read(0x21); + outb(LOCK_KEY, CFG_PORT); + return (msb * 256 + lsb); +} + +static int se10_wdt_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct se10_wdt *priv; + unsigned int chip_id; + int ret; + + if (!request_muxed_region(CFG_PORT, CFG_SIZE, DRVNAME)) + return -EBUSY; + + chip_id = get_chipID(); + if (chip_id != 0x5632) { + release_region(CFG_PORT, CFG_SIZE); + return -ENODEV; + } + + lpc_write(CFG_LDN, CFG_BRAM_LDN); + bram_base = (lpc_read(0x60) << 8) | lpc_read(0x61); + release_region(CFG_PORT, CFG_SIZE); + + dev_info(dev, "Found Lenovo SE10 0x%x\n", chip_id); + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + watchdog_set_drvdata(&priv->wdd, priv); + + priv->wdd.parent = dev; + priv->wdd.info = &wdt_info, + priv->wdd.ops = &se10_wdt_ops, + priv->wdd.timeout = WATCHDOG_TIMEOUT; /* Set default timeout */ + priv->wdd.min_timeout = MIN_TIMEOUT; + priv->wdd.max_timeout = MAX_TIMEOUT; + + set_bram(CUS_WDT_CFG, WATCHDOG_TIMEOUT); /* Set time to default */ + + watchdog_init_timeout(&priv->wdd, timeout, dev); + watchdog_set_nowayout(&priv->wdd, nowayout); + watchdog_stop_on_reboot(&priv->wdd); + watchdog_stop_on_unregister(&priv->wdd); + + ret = devm_watchdog_register_device(dev, &priv->wdd); + + dev_dbg(&pdev->dev, "initialized. timeout=%d sec (nowayout=%d)\n", + priv->wdd.timeout, nowayout); + + return ret; +} + +static struct platform_driver se10_wdt_driver = { + .driver = { + .name = DRVNAME, + }, + .probe = se10_wdt_probe, +}; + +static int se10_create_platform_device(const struct dmi_system_id *id) +{ + int err; + + se10_pdev = platform_device_alloc("lenovo-se10-wdt", -1); + if (!se10_pdev) + return -ENOMEM; + + err = platform_device_add(se10_pdev); + if (err) + platform_device_put(se10_pdev); + + return err; +} + +static const struct dmi_system_id se10_dmi_table[] __initconst = { + { + .ident = "LENOVO-SE10", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "12NH"), + }, + .callback = se10_create_platform_device, + }, + { + .ident = "LENOVO-SE10", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "12NJ"), + }, + .callback = se10_create_platform_device, + }, + { + .ident = "LENOVO-SE10", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "12NK"), + }, + .callback = se10_create_platform_device, + }, + { + .ident = "LENOVO-SE10", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "12NL"), + }, + .callback = se10_create_platform_device, + }, + { + .ident = "LENOVO-SE10", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "12NM"), + }, + .callback = se10_create_platform_device, + }, + {} +}; +MODULE_DEVICE_TABLE(dmi, se10_dmi_table); + +static int __init se10_wdt_init(void) +{ + if (!dmi_check_system(se10_dmi_table)) + return -ENODEV; + + return platform_driver_register(&se10_wdt_driver); +} + +static void __exit se10_wdt_exit(void) +{ + if (se10_pdev) + platform_device_unregister(se10_pdev); + platform_driver_unregister(&se10_wdt_driver); +} + +module_init(se10_wdt_init); +module_exit(se10_wdt_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("David Ober<dober@lenovo.com>"); +MODULE_AUTHOR("Mark Pearson <mpearson-lenovo@squebb.ca>"); +MODULE_DESCRIPTION("WDT driver for Lenovo SE10"); diff --git a/drivers/watchdog/mtx-1_wdt.c b/drivers/watchdog/mtx-1_wdt.c index 152e41ecbb14..06756135033d 100644 --- a/drivers/watchdog/mtx-1_wdt.c +++ b/drivers/watchdog/mtx-1_wdt.c @@ -236,7 +236,6 @@ static struct platform_driver mtx1_wdt_driver = { .probe = mtx1_wdt_probe, .remove_new = mtx1_wdt_remove, .driver.name = "mtx1-wdt", - .driver.owner = THIS_MODULE, }; module_platform_driver(mtx1_wdt_driver); diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c index 0fe71f7e66d5..52d49e4e35a0 100644 --- a/drivers/watchdog/octeon-wdt-main.c +++ b/drivers/watchdog/octeon-wdt-main.c @@ -381,11 +381,7 @@ static int octeon_wdt_cpu_online(unsigned int cpu) /* Must set the irq affinity here */ if (octeon_has_feature(OCTEON_FEATURE_CIU3)) { - cpumask_t mask; - - cpumask_clear(&mask); - cpumask_set_cpu(cpu, &mask); - irq_set_affinity(irq, &mask); + irq_set_affinity(irq, cpumask_of(cpu)); } cpumask_set_cpu(cpu, &irq_enabled_cpus); diff --git a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c index 9215793a1c81..4895a69015a8 100644 --- a/drivers/watchdog/rti_wdt.c +++ b/drivers/watchdog/rti_wdt.c @@ -59,6 +59,8 @@ #define PON_REASON_EOF_NUM 0xCCCCBBBB #define RESERVED_MEM_MIN_SIZE 12 +#define MAX_HW_ERROR 250 + static int heartbeat = DEFAULT_HEARTBEAT; /* @@ -97,7 +99,7 @@ static int rti_wdt_start(struct watchdog_device *wdd) * to be 50% or less than that; we obviouly want to configure the open * window as large as possible so we select the 50% option. */ - wdd->min_hw_heartbeat_ms = 500 * wdd->timeout; + wdd->min_hw_heartbeat_ms = 520 * wdd->timeout + MAX_HW_ERROR; /* Generate NMI when wdt expires */ writel_relaxed(RTIWWDRX_NMI, wdt->base + RTIWWDRXCTRL); @@ -131,31 +133,33 @@ static int rti_wdt_setup_hw_hb(struct watchdog_device *wdd, u32 wsize) * be petted during the open window; not too early or not too late. * The HW configuration options only allow for the open window size * to be 50% or less than that. + * To avoid any glitches, we accommodate 2% + max hardware error + * safety margin. */ switch (wsize) { case RTIWWDSIZE_50P: - /* 50% open window => 50% min heartbeat */ - wdd->min_hw_heartbeat_ms = 500 * heartbeat; + /* 50% open window => 52% min heartbeat */ + wdd->min_hw_heartbeat_ms = 520 * heartbeat + MAX_HW_ERROR; break; case RTIWWDSIZE_25P: - /* 25% open window => 75% min heartbeat */ - wdd->min_hw_heartbeat_ms = 750 * heartbeat; + /* 25% open window => 77% min heartbeat */ + wdd->min_hw_heartbeat_ms = 770 * heartbeat + MAX_HW_ERROR; break; case RTIWWDSIZE_12P5: - /* 12.5% open window => 87.5% min heartbeat */ - wdd->min_hw_heartbeat_ms = 875 * heartbeat; + /* 12.5% open window => 89.5% min heartbeat */ + wdd->min_hw_heartbeat_ms = 895 * heartbeat + MAX_HW_ERROR; break; case RTIWWDSIZE_6P25: - /* 6.5% open window => 93.5% min heartbeat */ - wdd->min_hw_heartbeat_ms = 935 * heartbeat; + /* 6.5% open window => 95.5% min heartbeat */ + wdd->min_hw_heartbeat_ms = 955 * heartbeat + MAX_HW_ERROR; break; case RTIWWDSIZE_3P125: - /* 3.125% open window => 96.9% min heartbeat */ - wdd->min_hw_heartbeat_ms = 969 * heartbeat; + /* 3.125% open window => 98.9% min heartbeat */ + wdd->min_hw_heartbeat_ms = 989 * heartbeat + MAX_HW_ERROR; break; default: @@ -233,14 +237,6 @@ static int rti_wdt_probe(struct platform_device *pdev) return -EINVAL; } - /* - * If watchdog is running at 32k clock, it is not accurate. - * Adjust frequency down in this case so that we don't pet - * the watchdog too often. - */ - if (wdt->freq < 32768) - wdt->freq = wdt->freq * 9 / 10; - pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); if (ret < 0) { diff --git a/drivers/watchdog/sa1100_wdt.c b/drivers/watchdog/sa1100_wdt.c index 5d2df008b92a..34a917221e31 100644 --- a/drivers/watchdog/sa1100_wdt.c +++ b/drivers/watchdog/sa1100_wdt.c @@ -191,9 +191,8 @@ static int sa1100dog_probe(struct platform_device *pdev) if (!res) return -ENXIO; reg_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); - ret = PTR_ERR_OR_ZERO(reg_base); - if (ret) - return ret; + if (!reg_base) + return -ENOMEM; clk = clk_get(NULL, "OSTIMER0"); if (IS_ERR(clk)) { diff --git a/drivers/xen/grant-dma-ops.c b/drivers/xen/grant-dma-ops.c index 76f6f26265a3..29257d2639db 100644 --- a/drivers/xen/grant-dma-ops.c +++ b/drivers/xen/grant-dma-ops.c @@ -282,7 +282,7 @@ static int xen_grant_dma_supported(struct device *dev, u64 mask) static const struct dma_map_ops xen_grant_dma_ops = { .alloc = xen_grant_dma_alloc, .free = xen_grant_dma_free, - .alloc_pages = xen_grant_dma_alloc_pages, + .alloc_pages_op = xen_grant_dma_alloc_pages, .free_pages = xen_grant_dma_free_pages, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, diff --git a/drivers/xen/pvcalls-back.c b/drivers/xen/pvcalls-back.c index d52593466a79..fd7ed65e0197 100644 --- a/drivers/xen/pvcalls-back.c +++ b/drivers/xen/pvcalls-back.c @@ -517,6 +517,10 @@ static void __pvcalls_back_accept(struct work_struct *work) { struct sockpass_mapping *mappass = container_of( work, struct sockpass_mapping, register_work); + struct proto_accept_arg arg = { + .flags = O_NONBLOCK, + .kern = true, + }; struct sock_mapping *map; struct pvcalls_ioworker *iow; struct pvcalls_fedata *fedata; @@ -548,7 +552,7 @@ static void __pvcalls_back_accept(struct work_struct *work) sock->type = mappass->sock->type; sock->ops = mappass->sock->ops; - ret = inet_accept(mappass->sock, sock, O_NONBLOCK, true); + ret = inet_accept(mappass->sock, sock, &arg); if (ret == -EAGAIN) { sock_release(sock); return; diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 0e6c6c25d154..6579ae3f6dac 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -216,7 +216,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, */ trace_swiotlb_bounced(dev, dev_addr, size); - map = swiotlb_tbl_map_single(dev, phys, size, size, 0, dir, attrs); + map = swiotlb_tbl_map_single(dev, phys, size, 0, dir, attrs); if (map == (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; @@ -403,7 +403,7 @@ const struct dma_map_ops xen_swiotlb_dma_ops = { .dma_supported = xen_swiotlb_dma_supported, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, .max_mapping_size = swiotlb_max_mapping_size, }; |